Add support for large integer literals and fix a bug in abs/round

minijinja/src/compiler/codegen.rs

@@ -6,6 +6,7 @@ use crate::compiler::instructions::{
 };
 use crate::compiler::tokens::Span;
 use crate::output::CaptureMode;
+use crate::value::ops::neg;
 use crate::value::Value;
 
 #[cfg(test)]
@@ -562,11 +563,25 @@ impl<'source> CodeGenerator<'source> {
             }
             ast::Expr::UnaryOp(c) => {
                 self.set_line_from_span(c.span());
-                self.compile_expr(&c.expr);
                 match c.op {
-                    ast::UnaryOpKind::Not => self.add(Instruction::Not),
-                    ast::UnaryOpKind::Neg => self.add_with_span(Instruction::Neg, c.span()),
-                };
+                    ast::UnaryOpKind::Not => {
+                        self.compile_expr(&c.expr);
+                        self.add(Instruction::Not);
+                    }
+                    ast::UnaryOpKind::Neg => {
+                        // common case: negative numbers.  In that case we
+                        // directly negate them if this is possible without
+                        // an error.
+                        if let ast::Expr::Const(ref c) = c.expr {
+                            if let Ok(negated) = neg(&c.value) {
+                                self.add(Instruction::LoadConst(negated));
+                                return;
+                            }
+                        }
+                        self.compile_expr(&c.expr);
+                        self.add_with_span(Instruction::Neg, c.span());
+                    }
+                }
             }
             ast::Expr::BinOp(c) => {
                 self.compile_bin_op(c);

minijinja/src/compiler/lexer.rs

@@ -228,16 +228,17 @@ impl<'s> TokenizerState<'s> {
         let mut state = State::Integer;
         let mut num_len = self
             .rest
-            .chars()
+            .as_bytes()
+            .iter()
             .take_while(|&c| c.is_ascii_digit())
             .count();
-        for c in self.rest.chars().skip(num_len) {
+        for c in self.rest.as_bytes()[num_len..].iter().copied() {
             state = match (c, state) {
-                ('.', State::Integer) => State::Fraction,
-                ('E' | 'e', State::Integer | State::Fraction) => State::Exponent,
-                ('+' | '-', State::Exponent) => State::ExponentSign,
-                ('0'..='9', State::Exponent) => State::ExponentSign,
-                ('0'..='9', state) => state,
+                (b'.', State::Integer) => State::Fraction,
+                (b'E' | b'e', State::Integer | State::Fraction) => State::Exponent,
+                (b'+' | b'-', State::Exponent) => State::ExponentSign,
+                (b'0'..=b'9', State::Exponent) => State::ExponentSign,
+                (b'0'..=b'9', state) => state,
                 _ => break,
             };
             num_len += 1;
@@ -250,9 +251,11 @@ impl<'s> TokenizerState<'s> {
                 num.parse()
                     .map(Token::Float)
                     .map_err(|_| self.syntax_error("invalid float"))
+            } else if let Ok(int) = num.parse() {
+                Ok(Token::Int(int))
             } else {
                 num.parse()
-                    .map(Token::Int)
+                    .map(Token::Int128)
                     .map_err(|_| self.syntax_error("invalid integer"))
             }),
             self.span(old_loc),

minijinja/src/compiler/parser.rs

@@ -536,6 +536,7 @@ impl<'a> Parser<'a> {
             Token::Str(val) => Ok(const_val!(val)),
             Token::String(val) => Ok(const_val!(val)),
             Token::Int(val) => Ok(const_val!(val)),
+            Token::Int128(val) => Ok(const_val!(val)),
             Token::Float(val) => Ok(const_val!(val)),
             Token::ParenOpen => self.parse_tuple_or_expression(span),
             Token::BracketOpen => self.parse_list_expr(span),

minijinja/src/compiler/tokens.rs

@@ -25,7 +25,9 @@ pub enum Token<'a> {
     /// An allocated string.
     String(String),
     /// An integer (limited to i64)
-    Int(i64),
+    Int(u64),
+    /// A large integer
+    Int128(u128),
     /// A float
     Float(f64),
     /// A plus (`+`) operator.
@@ -92,7 +94,7 @@ impl<'a> fmt::Display for Token<'a> {
             Token::BlockEnd => f.write_str("end of block"),
             Token::Ident(_) => f.write_str("identifier"),
             Token::Str(_) | Token::String(_) => f.write_str("string"),
-            Token::Int(_) => f.write_str("integer"),
+            Token::Int(_) | Token::Int128(_) => f.write_str("integer"),
             Token::Float(_) => f.write_str("float"),
             Token::Plus => f.write_str("`+`"),
             Token::Minus => f.write_str("`-`"),

minijinja/src/filters.rs

@@ -552,6 +552,7 @@ mod builtins {
     #[cfg_attr(docsrs, doc(cfg(feature = "builtins")))]
     pub fn abs(value: Value) -> Result<Value, Error> {
         match value.0 {
+            ValueRepr::U64(_) | ValueRepr::U128(_) => Ok(value),
             ValueRepr::I64(x) => match x.checked_abs() {
                 Some(rv) => Ok(Value::from(rv)),
                 None => Ok(Value::from((x as i128).abs())), // this cannot overflow
@@ -595,14 +596,16 @@ mod builtins {
     #[cfg_attr(docsrs, doc(cfg(feature = "builtins")))]
     pub fn round(value: Value, precision: Option<i32>) -> Result<Value, Error> {
         match value.0 {
-            ValueRepr::I64(_) | ValueRepr::I128(_) => Ok(value),
+            ValueRepr::I64(_) | ValueRepr::I128(_) | ValueRepr::U64(_) | ValueRepr::U128(_) => {
+                Ok(value)
+            }
             ValueRepr::F64(val) => {
                 let x = 10f64.powi(precision.unwrap_or(0));
                 Ok(Value::from((x * val).round() / x))
             }
             _ => Err(Error::new(
                 ErrorKind::InvalidOperation,
-                "cannot round value",
+                format!("cannot round value ({})", value.kind()),
             )),
         }
     }

minijinja/src/value/ops.rs

@@ -3,6 +3,8 @@ use std::convert::{TryFrom, TryInto};
 use crate::error::{Error, ErrorKind};
 use crate::value::{KeyRef, ObjectKind, SeqObject, Value, ValueKind, ValueRepr};
 
+const MIN_I128_AS_POS_U128: u128 = 170141183460469231731687303715884105728;
+
 pub enum CoerceResult<'a> {
     I128(i128, i128),
     F64(f64, f64),
@@ -238,6 +240,11 @@ pub fn neg(val: &Value) -> Result<Value, Error> {
     if val.kind() == ValueKind::Number {
         match val.0 {
             ValueRepr::F64(x) => Ok((-x).into()),
+            // special case for the largest i128 that can still be
+            // represented.
+            ValueRepr::U128(x) if x.0 == MIN_I128_AS_POS_U128 => {
+                Ok(Value::from(MIN_I128_AS_POS_U128))
+            }
             _ => {
                 if let Ok(x) = i128::try_from(val.clone()) {
                     x.checked_mul(-1)

minijinja/tests/lexer-inputs/literals.txt

@@ -5,3 +5,6 @@
 {{ 2 }}
 {{ [1, 2, 3] }}
 {{ {"foo": "bar"} }}
+{{ -9223372036854775808 }}
+{{ -170141183460469231731687303715884105728 }}
+{{ 170141183460469231731687303715884105727 }}

minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap

@@ -1,6 +1,6 @@
 ---
 source: minijinja/tests/test_lexer.rs
-description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}"
+description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 2 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}"
 input_file: minijinja/tests/lexer-inputs/literals.txt
 ---
 VariableStart
@@ -79,4 +79,32 @@ VariableEnd
   "}}"
 TemplateData("\n")
   "\n"
+VariableStart
+  "{{"
+Minus
+  "-"
+Int(9223372036854775808)
+  "9223372036854775808"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Minus
+  "-"
+Int128(170141183460469231731687303715884105728)
+  "170141183460469231731687303715884105728"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
+VariableStart
+  "{{"
+Int128(170141183460469231731687303715884105727)
+  "170141183460469231731687303715884105727"
+VariableEnd
+  "}}"
+TemplateData("\n")
+  "\n"
 

minijinja/tests/test_value.rs

@@ -347,7 +347,7 @@ fn test_struct_object_downcast() {
     }
 
     impl StructObject for Thing {
-        fn get_field(&self, name: &str) -> Option<Value> {
+        fn get_field(&self, _name: &str) -> Option<Value> {
             None
         }
     }