Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tolerate underscores in number literals #443

Merged
merged 3 commits into from
Mar 23, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Small improvements
mitsuhiko committed Mar 23, 2024
commit 8a1202fbed4c5a6158f561fc61e5de34505ff782
23 changes: 14 additions & 9 deletions minijinja/src/compiler/lexer.rs
mitsuhiko marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use std::borrow::Cow;

use crate::compiler::tokens::{Span, Token};
use crate::error::{Error, ErrorKind};
use crate::utils::{memchr, memstr, unescape};
@@ -245,8 +247,9 @@ impl<'s> TokenizerState<'s> {
.rest
.as_bytes()
.iter()
.take_while(|&c| c.is_ascii_digit() || *c == b'_')
.take_while(|&c| c.is_ascii_digit())
.count();
let mut has_underscore = false;
for c in self.rest.as_bytes()[num_len..].iter().copied() {
state = match (c, state) {
(b'.', State::Integer) => State::Fraction,
@@ -255,22 +258,24 @@ impl<'s> TokenizerState<'s> {
(b'0'..=b'9', State::Exponent) => State::ExponentSign,
(b'0'..=b'9', state) => state,
(b'a'..=b'f' | b'A'..=b'F', State::RadixInteger) if radix == 16 => state,
(
b'_',
State::RadixInteger | State::Integer | State::Fraction | State::Exponent,
) => state,
(b'_', _) => {
has_underscore = true;
state
}
_ => break,
};
num_len += 1;
}
let is_float = !matches!(state, State::Integer | State::RadixInteger);

if num_len > 0 && self.rest.as_bytes()[num_len - 1] == b'_' {
return Result::Err(self.syntax_error("'_' may not occur at end of number"));
let mut num = Cow::Borrowed(self.advance(num_len));
if has_underscore {
if num.ends_with('_') {
return Err(self.syntax_error("'_' may not occur at end of number"));
}
num = Cow::Owned(num.replace('_', ""));
}

let num = self.advance(num_len).replace('_', "");

Ok((
ok!(if is_float {
num.parse()
3 changes: 3 additions & 0 deletions minijinja/tests/inputs/err_bad_trailing_underscore.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{}
---
{{ 10_ }}
4 changes: 4 additions & 0 deletions minijinja/tests/lexer-inputs/literals.txt
Original file line number Diff line number Diff line change
@@ -20,3 +20,7 @@
{{ 0xdeadbeef }}
{{ 0XDEADBEEF }}
{{ 0XDEAD_BEEF }}
{{ 1e2 }}
{{ 1e_2 }}
{{ 1_1e+_2 }}
{{ 1_1e-_2 }}
36 changes: 33 additions & 3 deletions minijinja/tests/snapshots/test_lexer__lexer@literals.txt.snap
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
---
source: minijinja/tests/test_lexer.rs
assertion_line: 24
description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 1_000.0 }}\n{{ 2 }}\n{{ 2_000 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -9_223_372_036_854_775_808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0O7_77 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}\n{{ 0XDEAD_BEEF }}"
description: "{{ true }}\n{{ false }}\n{{ null }}\n{{ 1.0 }}\n{{ 1_000.0 }}\n{{ 2 }}\n{{ 2_000 }}\n{{ [1, 2, 3] }}\n{{ {\"foo\": \"bar\"} }}\n{{ -9223372036854775808 }}\n{{ -9_223_372_036_854_775_808 }}\n{{ -170141183460469231731687303715884105728 }}\n{{ 170141183460469231731687303715884105727 }}\n{{ 340282366920938463463374607431768211455 }}\n{{ 0b1111 }}\n{{ 0B1001 }}\n{{ 0o777 }}\n{{ 0O777 }}\n{{ 0O7_77 }}\n{{ 0xdeadbeef }}\n{{ 0XDEADBEEF }}\n{{ 0XDEAD_BEEF }}\n{{ 1e2 }}\n{{ 1e_2 }}\n{{ 1_1e+_2 }}\n{{ 1_1e-_2 }}"
input_file: minijinja/tests/lexer-inputs/literals.txt
---
VariableStart
@@ -206,4 +205,35 @@ VariableEnd
"}}"
TemplateData("\n")
"\n"

VariableStart
"{{"
Float(100.0)
"1e2"
VariableEnd
"}}"
TemplateData("\n")
"\n"
VariableStart
"{{"
Float(100.0)
"1e_2"
VariableEnd
"}}"
TemplateData("\n")
"\n"
VariableStart
"{{"
Float(1100.0)
"1_1e+_2"
VariableEnd
"}}"
TemplateData("\n")
"\n"
VariableStart
"{{"
Float(0.11)
"1_1e-_2"
VariableEnd
"}}"
TemplateData("\n")
"\n"
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
source: minijinja/tests/test_templates.rs
description: "{{ 10_ }}"
info: {}
input_file: minijinja/tests/inputs/err_bad_trailing_underscore.txt
---
!!!SYNTAX ERROR!!!

Error {
kind: SyntaxError,
detail: "'_' may not occur at end of number",
name: "err_bad_trailing_underscore.txt",
line: 1,
}

syntax error: '_' may not occur at end of number (in err_bad_trailing_underscore.txt:1)
----------------------- err_bad_trailing_underscore.txt -----------------------
1 > {{ 10_ }}
i ^^ syntax error
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
No referenced variables
-------------------------------------------------------------------------------