From 8fc23a58082a95224c904892da501fbd93dcb5f2 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Fri, 29 Jan 2021 12:14:13 +0100 Subject: [PATCH 1/9] Fix bug with escaped squiglies at the end of strigs Signed-off-by: Heinz N. Gies --- tests/script.rs | 2 ++ tests/scripts/string_interpolation_regexp/in.xz | Bin 0 -> 88 bytes tests/scripts/string_interpolation_regexp/out.xz | Bin 0 -> 76 bytes .../string_interpolation_regexp/script.tremor | 2 ++ tests/scripts/string_interpolation_tailing/in.xz | Bin 0 -> 60 bytes tests/scripts/string_interpolation_tailing/out.xz | Bin 0 -> 64 bytes .../string_interpolation_tailing/script.tremor | 1 + tremor-script/src/grammar.lalrpop | 3 +++ 8 files changed, 8 insertions(+) create mode 100644 tests/scripts/string_interpolation_regexp/in.xz create mode 100644 tests/scripts/string_interpolation_regexp/out.xz create mode 100644 tests/scripts/string_interpolation_regexp/script.tremor create mode 100644 tests/scripts/string_interpolation_tailing/in.xz create mode 100644 tests/scripts/string_interpolation_tailing/out.xz create mode 100644 tests/scripts/string_interpolation_tailing/script.tremor diff --git a/tests/script.rs b/tests/script.rs index 68708f8f16..98dc676fcb 100644 --- a/tests/script.rs +++ b/tests/script.rs @@ -153,6 +153,8 @@ test_cases!( // TODO // const_in_const_lookup, // INSERT +string_interpolation_tailing, +string_interpolation_regexp, binary_binary, binary_binary_string, binary_string_binary, diff --git a/tests/scripts/string_interpolation_regexp/in.xz b/tests/scripts/string_interpolation_regexp/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..078b4872c99f83fe1aff15ad683ea8b66289133a GIT binary patch literal 88 zcmV-e0H^=`H+ooF000E$*0e?f03iVu0001VFXf})0384vT>udb0As5Mg$`Ggqsi~? uPpR#442KKwcAvb^U;qGAlz5M$z*t!T0XQDBG#1Stw)1@f000001X)^Ej2`v? literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_regexp/out.xz b/tests/scripts/string_interpolation_regexp/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..284242531cf45abe54419486fdc389a9d170e0d2 GIT binary patch literal 76 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8H5eE47rTBjJZs>Ou5Xs%(*PMEV&r$ f_#0aiwSN9$VAK)$!IjG_zwL7^BTx?mOJo!P+qn{j literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_regexp/script.tremor b/tests/scripts/string_interpolation_regexp/script.tremor new file mode 100644 index 0000000000..5448799fef --- /dev/null +++ b/tests/scripts/string_interpolation_regexp/script.tremor @@ -0,0 +1,2 @@ +use std::re; +re::replace_all("[0-9]\{4\}", event, "####") \ No newline at end of file diff --git a/tests/scripts/string_interpolation_tailing/in.xz b/tests/scripts/string_interpolation_tailing/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..df7e88e6beb08d6cecbd970448f2b80e0896840d GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=85j+?fb19hGRxN%{8VIMlw!K0x3o@v O+vi$Fpg03dWE21^5D~xt literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_tailing/out.xz b/tests/scripts/string_interpolation_tailing/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..04456844b4dae0b076ebbf704dad36a48627d0a5 GIT binary patch literal 64 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8CaF74QrLSfP9{syRY6|oA;4{QI74P S{t{XFZJ%oyf#M7-kx>AD(i3t3 literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_tailing/script.tremor b/tests/scripts/string_interpolation_tailing/script.tremor new file mode 100644 index 0000000000..f6ed7de51d --- /dev/null +++ b/tests/scripts/string_interpolation_tailing/script.tremor @@ -0,0 +1 @@ +"\{1\}" \ No newline at end of file diff --git a/tremor-script/src/grammar.lalrpop b/tremor-script/src/grammar.lalrpop index f3f5cf0ea4..38c43a3ec9 100644 --- a/tremor-script/src/grammar.lalrpop +++ b/tremor-script/src/grammar.lalrpop @@ -691,6 +691,9 @@ StrLitElements: StrLitElements<'input> = { }, => StrLitElements(vec![literal], vec![]), + "\\}" => StrLitElements(vec!["\\}".into()], vec![]), + "\\{" => StrLitElements(vec!["\\{".into()], vec![]), + "{" "}" => StrLitElements(vec!["{}".into()], vec![expr]), } From e932ccb9005acc86f4f5c236fe74852bb926bb16 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Fri, 29 Jan 2021 14:32:25 +0100 Subject: [PATCH 2/9] Remove unused unsafe impls Signed-off-by: Heinz N. Gies --- tremor-script/src/std_lib/stats.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tremor-script/src/std_lib/stats.rs b/tremor-script/src/std_lib/stats.rs index 30af2c2d6b..12e72b40fe 100644 --- a/tremor-script/src/std_lib/stats.rs +++ b/tremor-script/src/std_lib/stats.rs @@ -23,7 +23,6 @@ use hdrhistogram::Histogram; use sketches_ddsketch::{Config as DDSketchConfig, DDSketch}; use std::cmp::max; use std::f64; -use std::marker::Send; use std::ops::RangeInclusive; use std::u64; @@ -452,9 +451,6 @@ impl std::default::Default for Dds { } } -unsafe impl Send for Dds {} -unsafe impl Sync for Dds {} - impl TremorAggrFn for Dds { fn accumulate<'event>(&mut self, args: &[&Value<'event>]) -> FResult<()> { if let Some(vals) = args.get(1).and_then(|v| v.as_array()) { @@ -655,9 +651,6 @@ impl std::default::Default for Hdr { } } -unsafe impl Send for Hdr {} -unsafe impl Sync for Hdr {} - impl Hdr { fn max(&self) -> u64 { max(self.max, 4) From 3e013b436801672f7c46a9bb46ac9d796d966c77 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Tue, 2 Feb 2021 15:25:06 +0100 Subject: [PATCH 3/9] Implement new escape sequence Signed-off-by: Heinz N. Gies --- CHANGELOG.md | 6 + tests/queries/state/query.trickle | 2 +- tests/script.rs | 8 +- .../heredoc_interpolation_extractor/error.txt | 4 +- .../script.tremor | 2 +- .../heredoc_interpolation_ident/error.txt | 4 +- .../heredoc_interpolation_ident/script.tremor | 2 +- .../heredoc_interpolation_multiline/error.txt | 4 +- .../script.tremor | 2 +- .../lexer_heredoc_interpolation/error.txt | 6 +- .../lexer_heredoc_interpolation/script.tremor | 2 +- .../lexer_heredoc_interpolation2/error.txt | 6 +- .../script.tremor | 2 +- .../lexer_heredoc_interpolation3/error.txt | 6 +- .../script.tremor | 2 +- .../lexer_string_interpolation/error.txt | 6 +- .../lexer_string_interpolation/script.tremor | 2 +- .../lexer_string_interpolation2/error.txt | 6 +- .../lexer_string_interpolation2/script.tremor | 2 +- .../lexer_string_interpolation3/error.txt | 4 +- .../lexer_string_interpolation3/script.tremor | 2 +- .../string_interpolation_eof/error.txt | 6 +- .../string_interpolation_eof/script.tremor | 2 +- .../string_interpolation_escape/error.txt | 4 +- .../string_interpolation_escape/script.tremor | 2 +- .../string_interpolation_extractor/error.txt | 4 +- .../script.tremor | 2 +- .../error.txt | 4 +- .../script.tremor | 2 +- tests/scripts/const_of_const/script.tremor | 2 +- tests/scripts/fn_extractors/script.tremor | 2 +- tests/scripts/fn_nest2_abs_fib/script.tremor | 6 +- tests/scripts/fn_nest2_fib/script.tremor | 4 +- .../heredoc_interpolation/script.tremor | 2 +- .../script.tremor | 2 +- tests/scripts/heredoc_quoted_curly/out.xz | Bin 116 -> 116 bytes .../heredoc_quoted_curly/script.tremor | 2 +- .../scripts/heredoc_regression/script.tremor | 4 +- .../heredoc_usefn_interpolation/script.tremor | 2 +- tests/scripts/new.sh | 5 + tests/scripts/pp_alias0/script.tremor | 2 +- tests/scripts/pp_alias1/script.tremor | 2 +- tests/scripts/pp_alias2/script.tremor | 2 +- tests/scripts/pp_alias3/script.tremor | 2 +- tests/scripts/pp_inline_nest1/script.tremor | 2 +- tests/scripts/pp_nest0/script.tremor | 2 +- tests/scripts/pp_nest1/script.tremor | 2 +- tests/scripts/pp_nest2/script.tremor | 2 +- tests/scripts/pp_nest3/script.tremor | 2 +- .../string_interpolation/script.tremor | 10 +- .../string_interpolation_escaped/in.xz | Bin 0 -> 60 bytes .../string_interpolation_escaped/out.xz | Bin 0 -> 100 bytes .../script.tremor | 5 + .../string_interpolation_escaped_hash/in.xz | Bin 0 -> 60 bytes .../string_interpolation_escaped_hash/out.xz | Bin 0 -> 80 bytes .../script.tremor | 6 + .../string_interpolation_import/script.tremor | 2 +- .../scripts/string_interpolation_nested/in.xz | Bin 0 -> 60 bytes .../string_interpolation_nested/out.xz | Bin 0 -> 60 bytes .../string_interpolation_nested/script.tremor | 1 + .../string_interpolation_prefix/script.tremor | 2 +- .../string_interpolation_quotes/script.tremor | 2 +- .../string_interpolation_regexp/out.xz | Bin 76 -> 88 bytes .../string_interpolation_regexp/script.tremor | 2 +- .../scripts/string_interpolation_simple/in.xz | Bin 0 -> 76 bytes .../string_interpolation_simple/out.xz | Bin 0 -> 92 bytes .../string_interpolation_simple/script.tremor | 1 + .../script.tremor | 2 +- tests/scripts/string_quoted_curly/out.xz | Bin 116 -> 112 bytes .../scripts/string_quoted_curly/script.tremor | 2 +- tremor-script/src/ast.rs | 23 + tremor-script/src/ast/base_expr.rs | 3 + tremor-script/src/ast/raw.rs | 132 ++- tremor-script/src/errors.rs | 2 +- tremor-script/src/grammar.lalrpop | 55 +- tremor-script/src/interpreter/imut_expr.rs | 21 +- tremor-script/src/lexer.rs | 886 +++++++----------- 77 files changed, 589 insertions(+), 723 deletions(-) create mode 100644 tests/scripts/string_interpolation_escaped/in.xz create mode 100644 tests/scripts/string_interpolation_escaped/out.xz create mode 100644 tests/scripts/string_interpolation_escaped/script.tremor create mode 100644 tests/scripts/string_interpolation_escaped_hash/in.xz create mode 100644 tests/scripts/string_interpolation_escaped_hash/out.xz create mode 100644 tests/scripts/string_interpolation_escaped_hash/script.tremor create mode 100644 tests/scripts/string_interpolation_nested/in.xz create mode 100644 tests/scripts/string_interpolation_nested/out.xz create mode 100644 tests/scripts/string_interpolation_nested/script.tremor create mode 100644 tests/scripts/string_interpolation_simple/in.xz create mode 100644 tests/scripts/string_interpolation_simple/out.xz create mode 100644 tests/scripts/string_interpolation_simple/script.tremor diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d5e847f8..40f0cc00ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Latest +### Breaking Changes + +* String interpolation is now done via `#{..}` instead of `{}`. `{` no longer needs to be escaped, but `\#{` needs an escape for literal `#{`. + ### New features * Default to thin-lto for all builds (prior this was only done in docker) @@ -11,6 +15,8 @@ * Statically link openssl * elastic sink now supports linked transports [#715](https://github.com/tremor-rs/tremor-runtime/pull/715) +### Fixes +* rewrite string interpolation to fix [#726](https://github.com/tremor-rs/tremor-runtime/issues/726) ## 0.9.4 ### New features diff --git a/tests/queries/state/query.trickle b/tests/queries/state/query.trickle index 0c50c9b238..8f48ab376c 100644 --- a/tests/queries/state/query.trickle +++ b/tests/queries/state/query.trickle @@ -10,7 +10,7 @@ script let state.count = state.count + 1 end; - { "e": event, "{args.name}": state.count } + { "e": event, "#{args.name}": state.count } end; create script seq1 from countable with name = "s1" end; diff --git a/tests/script.rs b/tests/script.rs index 98dc676fcb..f4ed1ca838 100644 --- a/tests/script.rs +++ b/tests/script.rs @@ -153,8 +153,12 @@ test_cases!( // TODO // const_in_const_lookup, // INSERT -string_interpolation_tailing, -string_interpolation_regexp, + string_interpolation_nested, + string_interpolation_escaped_hash, + string_interpolation_escaped, + string_interpolation_simple, + string_interpolation_tailing, + string_interpolation_regexp, binary_binary, binary_binary_string, binary_string_binary, diff --git a/tests/script_errors/heredoc_interpolation_extractor/error.txt b/tests/script_errors/heredoc_interpolation_extractor/error.txt index 1a9b2716c2..95d8bd528d 100644 --- a/tests/script_errors/heredoc_interpolation_extractor/error.txt +++ b/tests/script_errors/heredoc_interpolation_extractor/error.txt @@ -1,4 +1,4 @@ Error: 1 | let heredoc = """ - 2 | { | } - | ^^^ It looks like you forgot to terminate an extractor with a closing '|' \ No newline at end of file + 2 | #{ | } + | ^^^ It looks like you forgot to terminate an extractor with a closing '|' \ No newline at end of file diff --git a/tests/script_errors/heredoc_interpolation_extractor/script.tremor b/tests/script_errors/heredoc_interpolation_extractor/script.tremor index e0d4d5145d..c5576a2d8f 100644 --- a/tests/script_errors/heredoc_interpolation_extractor/script.tremor +++ b/tests/script_errors/heredoc_interpolation_extractor/script.tremor @@ -1,3 +1,3 @@ let heredoc = """ -{ | } +#{ | } """; diff --git a/tests/script_errors/heredoc_interpolation_ident/error.txt b/tests/script_errors/heredoc_interpolation_ident/error.txt index f784fc30cc..5691be8732 100644 --- a/tests/script_errors/heredoc_interpolation_ident/error.txt +++ b/tests/script_errors/heredoc_interpolation_ident/error.txt @@ -1,4 +1,4 @@ Error: 1 | """ - 2 | before { `ident unfinished } after - | ^^^^^^^^^^^^^^^^^^^^^^^^^ It looks like you forgot to terminate an ident with a closing '`' \ No newline at end of file + 2 | before #{ `ident unfinished } after + | ^^^^^^^^^^^^^^^^^^^^^^^^^ It looks like you forgot to terminate an ident with a closing '`' \ No newline at end of file diff --git a/tests/script_errors/heredoc_interpolation_ident/script.tremor b/tests/script_errors/heredoc_interpolation_ident/script.tremor index 5ad11f8b3c..ac33e00349 100644 --- a/tests/script_errors/heredoc_interpolation_ident/script.tremor +++ b/tests/script_errors/heredoc_interpolation_ident/script.tremor @@ -1,3 +1,3 @@ """ -before { `ident unfinished } after +before #{ `ident unfinished } after """ diff --git a/tests/script_errors/heredoc_interpolation_multiline/error.txt b/tests/script_errors/heredoc_interpolation_multiline/error.txt index 0dec0e186a..02415de2fe 100644 --- a/tests/script_errors/heredoc_interpolation_multiline/error.txt +++ b/tests/script_errors/heredoc_interpolation_multiline/error.txt @@ -1,4 +1,4 @@ Error: 1 | """ - 2 | { " - | ^ It looks like you forgot to terminate a string with a closing '"' \ No newline at end of file + 2 | #{ " + | ^ It looks like you forgot to terminate a string with a closing '"' \ No newline at end of file diff --git a/tests/script_errors/heredoc_interpolation_multiline/script.tremor b/tests/script_errors/heredoc_interpolation_multiline/script.tremor index d21e01fbcf..6a764f4fe1 100644 --- a/tests/script_errors/heredoc_interpolation_multiline/script.tremor +++ b/tests/script_errors/heredoc_interpolation_multiline/script.tremor @@ -1,5 +1,5 @@ """ -{ " +#{ " } """ diff --git a/tests/script_errors/lexer_heredoc_interpolation/error.txt b/tests/script_errors/lexer_heredoc_interpolation/error.txt index b0254dfcaa..f910e7b25d 100644 --- a/tests/script_errors/lexer_heredoc_interpolation/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation/error.txt @@ -1,5 +1,5 @@ Error: 1 | """ - 2 | { - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 2 | #{ + | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_heredoc_interpolation/script.tremor b/tests/script_errors/lexer_heredoc_interpolation/script.tremor index 58a6a75af6..dfbdebd5a2 100644 --- a/tests/script_errors/lexer_heredoc_interpolation/script.tremor +++ b/tests/script_errors/lexer_heredoc_interpolation/script.tremor @@ -1,2 +1,2 @@ """ -{ +#{ diff --git a/tests/script_errors/lexer_heredoc_interpolation2/error.txt b/tests/script_errors/lexer_heredoc_interpolation2/error.txt index 9381d09df7..9d422d6074 100644 --- a/tests/script_errors/lexer_heredoc_interpolation2/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation2/error.txt @@ -1,5 +1,5 @@ Error: 1 | """ - 2 | {""" - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 2 | #{""" + | ^^^^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_heredoc_interpolation2/script.tremor b/tests/script_errors/lexer_heredoc_interpolation2/script.tremor index 909ec6ff46..01f2083454 100644 --- a/tests/script_errors/lexer_heredoc_interpolation2/script.tremor +++ b/tests/script_errors/lexer_heredoc_interpolation2/script.tremor @@ -1,2 +1,2 @@ """ -{""" +#{""" diff --git a/tests/script_errors/lexer_heredoc_interpolation3/error.txt b/tests/script_errors/lexer_heredoc_interpolation3/error.txt index cf99485e41..998c0aec51 100644 --- a/tests/script_errors/lexer_heredoc_interpolation3/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation3/error.txt @@ -1,5 +1,5 @@ Error: 2 | let x = """ - 3 | {"""; - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 3 | #{"""; + | ^^^^^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_heredoc_interpolation3/script.tremor b/tests/script_errors/lexer_heredoc_interpolation3/script.tremor index 01fffd98a9..9a0dd24b80 100644 --- a/tests/script_errors/lexer_heredoc_interpolation3/script.tremor +++ b/tests/script_errors/lexer_heredoc_interpolation3/script.tremor @@ -1,3 +1,3 @@ use std::string; let x = """ -{"""; +#{"""; diff --git a/tests/script_errors/lexer_string_interpolation/error.txt b/tests/script_errors/lexer_string_interpolation/error.txt index 833d58e5a8..0df1d687ff 100644 --- a/tests/script_errors/lexer_string_interpolation/error.txt +++ b/tests/script_errors/lexer_string_interpolation/error.txt @@ -1,4 +1,4 @@ Error: - 1 | "{" - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 1 | "#{" + | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_string_interpolation/script.tremor b/tests/script_errors/lexer_string_interpolation/script.tremor index 9acbf4477f..3674a0c1bb 100644 --- a/tests/script_errors/lexer_string_interpolation/script.tremor +++ b/tests/script_errors/lexer_string_interpolation/script.tremor @@ -1 +1 @@ -"{" +"#{" diff --git a/tests/script_errors/lexer_string_interpolation2/error.txt b/tests/script_errors/lexer_string_interpolation2/error.txt index 63e1fe3447..6c9b9e7ac1 100644 --- a/tests/script_errors/lexer_string_interpolation2/error.txt +++ b/tests/script_errors/lexer_string_interpolation2/error.txt @@ -1,4 +1,4 @@ Error: - 1 | let x = "before { { {}"; - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 1 | let x = "before #{ { {}"; + | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_string_interpolation2/script.tremor b/tests/script_errors/lexer_string_interpolation2/script.tremor index 1cbc345948..aa84044a9a 100644 --- a/tests/script_errors/lexer_string_interpolation2/script.tremor +++ b/tests/script_errors/lexer_string_interpolation2/script.tremor @@ -1,2 +1,2 @@ -let x = "before { { {}"; +let x = "before #{ { {}"; diff --git a/tests/script_errors/lexer_string_interpolation3/error.txt b/tests/script_errors/lexer_string_interpolation3/error.txt index 8a0d080bac..3d8bc5df18 100644 --- a/tests/script_errors/lexer_string_interpolation3/error.txt +++ b/tests/script_errors/lexer_string_interpolation3/error.txt @@ -1,4 +1,4 @@ Error: 1 | use std::string; - 2 | let e = "\u0065 { `unterminated_ident } \u0076 {2} \u0080"; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It looks like you forgot to terminate an ident with a closing '`' \ No newline at end of file + 2 | let e = "\u0065 #{ `unterminated_ident } \u0076 #{2} \u0080"; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It looks like you forgot to terminate an ident with a closing '`' \ No newline at end of file diff --git a/tests/script_errors/lexer_string_interpolation3/script.tremor b/tests/script_errors/lexer_string_interpolation3/script.tremor index 3fe3c2cb31..0f32c8f5fa 100644 --- a/tests/script_errors/lexer_string_interpolation3/script.tremor +++ b/tests/script_errors/lexer_string_interpolation3/script.tremor @@ -1,2 +1,2 @@ use std::string; -let e = "\u0065 { `unterminated_ident } \u0076 {2} \u0080"; +let e = "\u0065 #{ `unterminated_ident } \u0076 #{2} \u0080"; diff --git a/tests/script_errors/string_interpolation_eof/error.txt b/tests/script_errors/string_interpolation_eof/error.txt index 103f04c53d..d212cf6c17 100644 --- a/tests/script_errors/string_interpolation_eof/error.txt +++ b/tests/script_errors/string_interpolation_eof/error.txt @@ -1,4 +1,4 @@ Error: - 1 | let si = "before { - | ^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '{' ? Escape it as '\{' or '{{'. \ No newline at end of file + 1 | let si = "before #{ + | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' + | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_eof/script.tremor b/tests/script_errors/string_interpolation_eof/script.tremor index 2fdde3825d..cbc703b76b 100644 --- a/tests/script_errors/string_interpolation_eof/script.tremor +++ b/tests/script_errors/string_interpolation_eof/script.tremor @@ -1 +1 @@ -let si = "before { +let si = "before #{ diff --git a/tests/script_errors/string_interpolation_escape/error.txt b/tests/script_errors/string_interpolation_escape/error.txt index cc2b86b6bf..930d50bde5 100644 --- a/tests/script_errors/string_interpolation_escape/error.txt +++ b/tests/script_errors/string_interpolation_escape/error.txt @@ -1,3 +1,3 @@ Error: - 1 | let escape = " { "\z" } "; - | ^^ An unexpected escape code 'z' was found \ No newline at end of file + 1 | let escape = " #{ "\z" } "; + | ^^ An unexpected escape code 'z' was found \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_escape/script.tremor b/tests/script_errors/string_interpolation_escape/script.tremor index 81583fd28f..f88dceee5f 100644 --- a/tests/script_errors/string_interpolation_escape/script.tremor +++ b/tests/script_errors/string_interpolation_escape/script.tremor @@ -1 +1 @@ -let escape = " { "\z" } "; +let escape = " #{ "\z" } "; diff --git a/tests/script_errors/string_interpolation_extractor/error.txt b/tests/script_errors/string_interpolation_extractor/error.txt index 2fd86a2981..71cfb453b1 100644 --- a/tests/script_errors/string_interpolation_extractor/error.txt +++ b/tests/script_errors/string_interpolation_extractor/error.txt @@ -1,3 +1,3 @@ Error: - 1 | " { | } " - | ^^^^^ It looks like you forgot to terminate an extractor with a closing '|' \ No newline at end of file + 1 | " #{ | } " + | ^^^^^ It looks like you forgot to terminate an extractor with a closing '|' \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_extractor/script.tremor b/tests/script_errors/string_interpolation_extractor/script.tremor index 10f765d234..f96b6324c1 100644 --- a/tests/script_errors/string_interpolation_extractor/script.tremor +++ b/tests/script_errors/string_interpolation_extractor/script.tremor @@ -1 +1 @@ -" { | } " +" #{ | } " diff --git a/tests/script_errors/string_interpolation_invalid_utf8/error.txt b/tests/script_errors/string_interpolation_invalid_utf8/error.txt index f2bd7ac498..0ac5003eb3 100644 --- a/tests/script_errors/string_interpolation_invalid_utf8/error.txt +++ b/tests/script_errors/string_interpolation_invalid_utf8/error.txt @@ -1,3 +1,3 @@ Error: - 1 | " { " \u666G " } " - | ^^^^^^ An invalid UTF8 escape sequence was found \ No newline at end of file + 1 | " #{ " \u666G " } " + | ^^^^^^ An invalid UTF8 escape sequence was found \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_invalid_utf8/script.tremor b/tests/script_errors/string_interpolation_invalid_utf8/script.tremor index 5bbb9e7f9a..86a611c5af 100644 --- a/tests/script_errors/string_interpolation_invalid_utf8/script.tremor +++ b/tests/script_errors/string_interpolation_invalid_utf8/script.tremor @@ -1 +1 @@ -" { " \u666G " } " +" #{ " \u666G " } " diff --git a/tests/scripts/const_of_const/script.tremor b/tests/scripts/const_of_const/script.tremor index d62a55ab71..ca894a1a70 100644 --- a/tests/scripts/const_of_const/script.tremor +++ b/tests/scripts/const_of_const/script.tremor @@ -1,4 +1,4 @@ const a = "snot"; const b = ( a + " badger" ); -"{b}: {event}!" \ No newline at end of file +"#{b}: #{event}!" \ No newline at end of file diff --git a/tests/scripts/fn_extractors/script.tremor b/tests/scripts/fn_extractors/script.tremor index 2086909b70..8f71d0ae0f 100644 --- a/tests/scripts/fn_extractors/script.tremor +++ b/tests/scripts/fn_extractors/script.tremor @@ -3,7 +3,7 @@ use std::type; fn snottify(s) of case ("badger") => "snot badger, hell yea!" case (~ json||) => let s.snot = true, s - case (s) when type::is_string(s) => "snot {s}" + case (s) when type::is_string(s) => "snot #{s}" default => "snot caller, you can't snottify that!" end; diff --git a/tests/scripts/fn_nest2_abs_fib/script.tremor b/tests/scripts/fn_nest2_abs_fib/script.tremor index 12d11be83d..c65effb731 100644 --- a/tests/scripts/fn_nest2_abs_fib/script.tremor +++ b/tests/scripts/fn_nest2_abs_fib/script.tremor @@ -1,14 +1,14 @@ mod b with fn snot(n) with - "cake {n}" + "cake #{n}" end; end; mod a with mod b with - fn snot(n) with "snot {n}" end + fn snot(n) with "snot #{n}" end end; - fn badger(n) with "{b::snot(n)} badger {n}" end + fn badger(n) with "#{b::snot(n)} badger #{n}" end end; [a::badger(event), b::snot(event)] \ No newline at end of file diff --git a/tests/scripts/fn_nest2_fib/script.tremor b/tests/scripts/fn_nest2_fib/script.tremor index 554e6ccd1c..46e6192d24 100644 --- a/tests/scripts/fn_nest2_fib/script.tremor +++ b/tests/scripts/fn_nest2_fib/script.tremor @@ -1,8 +1,8 @@ mod a with mod b with - fn snot(n) with "snot {n}" end + fn snot(n) with "snot #{n}" end end; - fn badger(n) with "{b::snot(n)} badger {n}" end + fn badger(n) with "#{b::snot(n)} badger #{n}" end end; a::badger(event) \ No newline at end of file diff --git a/tests/scripts/heredoc_interpolation/script.tremor b/tests/scripts/heredoc_interpolation/script.tremor index 93f3bfb25c..33702e77b1 100644 --- a/tests/scripts/heredoc_interpolation/script.tremor +++ b/tests/scripts/heredoc_interpolation/script.tremor @@ -1,3 +1,3 @@ """ - {event.class} {event.answer} + #{event.class} #{event.answer} """ diff --git a/tests/scripts/heredoc_interpolation_quotes/script.tremor b/tests/scripts/heredoc_interpolation_quotes/script.tremor index 1da72a7f12..475a55a413 100644 --- a/tests/scripts/heredoc_interpolation_quotes/script.tremor +++ b/tests/scripts/heredoc_interpolation_quotes/script.tremor @@ -1,3 +1,3 @@ """ - "{event}" => {"tremolo"} + "#{event}" => #{"tremolo"} """ diff --git a/tests/scripts/heredoc_quoted_curly/out.xz b/tests/scripts/heredoc_quoted_curly/out.xz index c1f580be2b5a666de322647b920d84b3405f7fbb..1dd043d845c23c48a9e19c906c4e1491ac1fe88a 100644 GIT binary patch delta 95 zcmXRZnIILV&0rMEAjqNPR&RXaw8vl3Ua41t`@&M5U-F+aamuDjjq>@mLPfVDYt1Vr xaxg8Qc)ReE$jNz(3_vg`@0+aM+J#{ZjDBmFfc(3xH(zc%lKvaSW?+eo0ssR6BryO0 delta 95 zcmXRZnIIJ<&R`hJAjqNfE!F1K`R@03VyCoDyPdvg^Jl}Uhd0e|oIU@C-BHVz;R~uN wcPI3^nYypB6") diff --git a/tests/scripts/new.sh b/tests/scripts/new.sh index e4df9f45c6..0e164e5191 100755 --- a/tests/scripts/new.sh +++ b/tests/scripts/new.sh @@ -19,3 +19,8 @@ git add ${TARGET} sed -e '/^ \/\/ INSERT/a\ '"${NAME}," ${BASEDIR}/../script.rs > ${BASEDIR}/tmp && mv ${BASEDIR}/tmp ${BASEDIR}/../script.rs + +for f in ${TARGET}/* +do + echo "$f" +done \ No newline at end of file diff --git a/tests/scripts/pp_alias0/script.tremor b/tests/scripts/pp_alias0/script.tremor index 2dd84fe4a6..5b496c9e07 100644 --- a/tests/scripts/pp_alias0/script.tremor +++ b/tests/scripts/pp_alias0/script.tremor @@ -1,6 +1,6 @@ use foo as foo; match event of - case {} => "snot {foo::snot}" + case {} => "snot #{foo::snot}" default => "ko" end; diff --git a/tests/scripts/pp_alias1/script.tremor b/tests/scripts/pp_alias1/script.tremor index bad86ae57f..4ebeb66d0f 100644 --- a/tests/scripts/pp_alias1/script.tremor +++ b/tests/scripts/pp_alias1/script.tremor @@ -1,6 +1,6 @@ use foo :: bar as bar; match event of - case {} => "snot {bar::snot}" + case {} => "snot #{bar::snot}" default => "ko" end; diff --git a/tests/scripts/pp_alias2/script.tremor b/tests/scripts/pp_alias2/script.tremor index 99fbf02192..d35bee6f6f 100644 --- a/tests/scripts/pp_alias2/script.tremor +++ b/tests/scripts/pp_alias2/script.tremor @@ -1,6 +1,6 @@ use foo::bar::baz as baz; match event of - case {} => "snot {baz::snot}" + case {} => "snot #{baz::snot}" default => "ko" end; diff --git a/tests/scripts/pp_alias3/script.tremor b/tests/scripts/pp_alias3/script.tremor index fc978f6565..143ce9e76d 100644 --- a/tests/scripts/pp_alias3/script.tremor +++ b/tests/scripts/pp_alias3/script.tremor @@ -1,6 +1,6 @@ use foo:: bar ::baz :: mork as bork ; match event of - case {} => "snot {bork::snot}" + case {} => "snot #{bork::snot}" default => "ko" end; diff --git a/tests/scripts/pp_inline_nest1/script.tremor b/tests/scripts/pp_inline_nest1/script.tremor index 83932e6f97..3483da19b9 100644 --- a/tests/scripts/pp_inline_nest1/script.tremor +++ b/tests/scripts/pp_inline_nest1/script.tremor @@ -1,6 +1,6 @@ use foo; match event of - case {} => "snot {foo::bar::snot}" + case {} => "snot #{foo::bar::snot}" default => "ko" end; diff --git a/tests/scripts/pp_nest0/script.tremor b/tests/scripts/pp_nest0/script.tremor index a772392084..ad74c838ca 100644 --- a/tests/scripts/pp_nest0/script.tremor +++ b/tests/scripts/pp_nest0/script.tremor @@ -1,6 +1,6 @@ use foo; match event of - case {} => "snot {foo::snot}" + case {} => "snot #{foo::snot}" default => "ko" end; diff --git a/tests/scripts/pp_nest1/script.tremor b/tests/scripts/pp_nest1/script.tremor index 22cdc245b5..7626733126 100644 --- a/tests/scripts/pp_nest1/script.tremor +++ b/tests/scripts/pp_nest1/script.tremor @@ -1,6 +1,6 @@ use foo :: bar; match event of - case {} => "snot {bar::snot}" + case {} => "snot #{bar::snot}" default => "ko" end; diff --git a/tests/scripts/pp_nest2/script.tremor b/tests/scripts/pp_nest2/script.tremor index 068d6022c2..b7ac10f2a9 100644 --- a/tests/scripts/pp_nest2/script.tremor +++ b/tests/scripts/pp_nest2/script.tremor @@ -1,6 +1,6 @@ use foo::bar::baz; match event of - case {} => "snot {baz::snot}" + case {} => "snot #{baz::snot}" default => "ko" end; diff --git a/tests/scripts/pp_nest3/script.tremor b/tests/scripts/pp_nest3/script.tremor index cfb3b31aab..4653d947cd 100644 --- a/tests/scripts/pp_nest3/script.tremor +++ b/tests/scripts/pp_nest3/script.tremor @@ -1,6 +1,6 @@ use foo:: bar ::baz :: mork ; match event of - case {} => "snot {mork::snot}" + case {} => "snot #{mork::snot}" default => "ko" end; diff --git a/tests/scripts/string_interpolation/script.tremor b/tests/scripts/string_interpolation/script.tremor index cd2bc606e5..ad14d04c12 100644 --- a/tests/scripts/string_interpolation/script.tremor +++ b/tests/scripts/string_interpolation/script.tremor @@ -1,8 +1,8 @@ { - "pre_{event.class}": event.answer, - "{event.class}_post": event.answer + 1, - "aro_{event.class}_und": event.answer + 2, - "int_{event.class}_er_{event.answer}_lev_{event.answer}_ing": event.class, - "concatina_{event.class}{event.answer}{event.answer}{event.class}{event.class}_ing": event.class, + "pre_#{event.class}": event.answer, + "#{event.class}_post": event.answer + 1, + "aro_#{event.class}_und": event.answer + 2, + "int_#{event.class}_er_#{event.answer}_lev_#{event.answer}_ing": event.class, + "concatina_#{event.class}#{event.answer}#{event.answer}#{event.class}#{event.class}_ing": event.class, } \ No newline at end of file diff --git a/tests/scripts/string_interpolation_escaped/in.xz b/tests/scripts/string_interpolation_escaped/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..df7e88e6beb08d6cecbd970448f2b80e0896840d GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=85j+?fb19hGRxN%{8VIMlw!K0x3o@v O+vi$Fpg03dWE21^5D~xt literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_escaped/out.xz b/tests/scripts/string_interpolation_escaped/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..133f10701f3866a163a9624f7c2ae9ec69b73d59 GIT binary patch literal 100 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill`FxWDv$1>=4G^Br#I`;gRmby;Ut0u3K zIT!l3n_P_EukbxZ(Rfigr|kX{>qUTip0G3Zv`k=4U|@8%bKA1nN`BktT1KEa150ES E09XMe8~^|S literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_escaped/script.tremor b/tests/scripts/string_interpolation_escaped/script.tremor new file mode 100644 index 0000000000..092c4cb254 --- /dev/null +++ b/tests/scripts/string_interpolation_escaped/script.tremor @@ -0,0 +1,5 @@ + +[ + "\#{} this is a format string", + "\#{} this is a format #{"string"}", +] \ No newline at end of file diff --git a/tests/scripts/string_interpolation_escaped_hash/in.xz b/tests/scripts/string_interpolation_escaped_hash/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..df7e88e6beb08d6cecbd970448f2b80e0896840d GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=85j+?fb19hGRxN%{8VIMlw!K0x3o@v O+vi$Fpg03dWE21^5D~xt literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_escaped_hash/out.xz b/tests/scripts/string_interpolation_escaped_hash/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..a0fadd58954481e5fa121b96cb110f90be988dff GIT binary patch literal 80 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8APL%l$CUp>_CLF4VY!46w3vapT0Bf dT-cjC>lqmJ#KJg3@5^udT+0X)XJCnp0sv!l6@mZ& literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_escaped_hash/script.tremor b/tests/scripts/string_interpolation_escaped_hash/script.tremor new file mode 100644 index 0000000000..d8f1a0df83 --- /dev/null +++ b/tests/scripts/string_interpolation_escaped_hash/script.tremor @@ -0,0 +1,6 @@ +[ + "\#", + ">\#", + "\#<", + ">\#<", +] \ No newline at end of file diff --git a/tests/scripts/string_interpolation_import/script.tremor b/tests/scripts/string_interpolation_import/script.tremor index 48778e5505..3b3818592a 100644 --- a/tests/scripts/string_interpolation_import/script.tremor +++ b/tests/scripts/string_interpolation_import/script.tremor @@ -1,2 +1,2 @@ use std::random; -"{event} snot" \ No newline at end of file +"#{event} snot" \ No newline at end of file diff --git a/tests/scripts/string_interpolation_nested/in.xz b/tests/scripts/string_interpolation_nested/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..df7e88e6beb08d6cecbd970448f2b80e0896840d GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=85j+?fb19hGRxN%{8VIMlw!K0x3o@v O+vi$Fpg03dWE21^5D~xt literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_nested/out.xz b/tests/scripts/string_interpolation_nested/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..837ed000a6fe9ff827ad8d149c3f3d365d6487e4 GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8JLyKmADvwY};DM@?`E?21Xf{e4RN5 Q<+pvVWdy2WV2O+Z06ZfSWdHyG literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_nested/script.tremor b/tests/scripts/string_interpolation_nested/script.tremor new file mode 100644 index 0000000000..b4cddca482 --- /dev/null +++ b/tests/scripts/string_interpolation_nested/script.tremor @@ -0,0 +1 @@ +"#{"#{"#{7}"}"}" \ No newline at end of file diff --git a/tests/scripts/string_interpolation_prefix/script.tremor b/tests/scripts/string_interpolation_prefix/script.tremor index 88dbbcba46..ab42f001cc 100644 --- a/tests/scripts/string_interpolation_prefix/script.tremor +++ b/tests/scripts/string_interpolation_prefix/script.tremor @@ -1,3 +1,3 @@ """ -a "{event}" +a "#{event}" """ \ No newline at end of file diff --git a/tests/scripts/string_interpolation_quotes/script.tremor b/tests/scripts/string_interpolation_quotes/script.tremor index 240ab75eb2..f744b398d6 100644 --- a/tests/scripts/string_interpolation_quotes/script.tremor +++ b/tests/scripts/string_interpolation_quotes/script.tremor @@ -1,4 +1,4 @@ fn bla(e) with - " \"{e}\" \"{e}\" " + " \"#{e}\" \"#{e}\" " end; bla(event) diff --git a/tests/scripts/string_interpolation_regexp/out.xz b/tests/scripts/string_interpolation_regexp/out.xz index 284242531cf45abe54419486fdc389a9d170e0d2..9ad7560206ea3b18ce15ca4528c7e41462fc50c3 100644 GIT binary patch delta 69 zcmeZ?m>}WsfI*f)GL}J*haq`2dn;e~l*K3CzwnIK`p$RKRMWyod3Wz1#5Wy)p7WzJ>6Wy!^0$KTkJsP*#~1EY?}53XEh`E8$T L85w|pB{B*CL+K2L diff --git a/tests/scripts/string_interpolation_regexp/script.tremor b/tests/scripts/string_interpolation_regexp/script.tremor index 5448799fef..b7cddaab3b 100644 --- a/tests/scripts/string_interpolation_regexp/script.tremor +++ b/tests/scripts/string_interpolation_regexp/script.tremor @@ -1,2 +1,2 @@ use std::re; -re::replace_all("[0-9]\{4\}", event, "####") \ No newline at end of file +re::replace_all("[0-9]{4}", event, "####") \ No newline at end of file diff --git a/tests/scripts/string_interpolation_simple/in.xz b/tests/scripts/string_interpolation_simple/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..284242531cf45abe54419486fdc389a9d170e0d2 GIT binary patch literal 76 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8H5eE47rTBjJZs>Ou5Xs%(*PMEV&r$ f_#0aiwSN9$VAK)$!IjG_zwL7^BTx?mOJo!P+qn{j literal 0 HcmV?d00001 diff --git a/tests/scripts/string_interpolation_simple/out.xz b/tests/scripts/string_interpolation_simple/out.xz new file mode 100644 index 0000000000000000000000000000000000000000..0bf8389348ec6b361aed54c209b1873e6a97667b GIT binary patch literal 92 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill`FjzCl#xe-#{event}<" diff --git a/tests/scripts/string_interpolation_tailing/script.tremor b/tests/scripts/string_interpolation_tailing/script.tremor index f6ed7de51d..7993a507b5 100644 --- a/tests/scripts/string_interpolation_tailing/script.tremor +++ b/tests/scripts/string_interpolation_tailing/script.tremor @@ -1 +1 @@ -"\{1\}" \ No newline at end of file +"{1}" \ No newline at end of file diff --git a/tests/scripts/string_quoted_curly/out.xz b/tests/scripts/string_quoted_curly/out.xz index c1f580be2b5a666de322647b920d84b3405f7fbb..59135f986c8c0db12bf7807fec11ba6d9426658e 100644 GIT binary patch delta 92 zcmV-i0HgnOaF7{S0KEV(T>udXEJlAb(91~u6^$C_#&_>6T;Cp#0S%UeVi`-D9&IQ%OJ?1^DWip)cNlBcVefsPP?7HXY*&nsfRbsaGX8=huu-jm*ESl xDt9OJx|zDKu@zZw&Hw_Uc24g)cK7ozF#6750`g5W-&k)vlKvaSW?+eo0sy%4B}xDQ diff --git a/tests/scripts/string_quoted_curly/script.tremor b/tests/scripts/string_quoted_curly/script.tremor index a5cad97775..1afa0d20ea 100644 --- a/tests/scripts/string_quoted_curly/script.tremor +++ b/tests/scripts/string_quoted_curly/script.tremor @@ -1 +1 @@ -" \{event\} = {{event}} = {event}\n" +" \#{event} = #{event}\n" diff --git a/tremor-script/src/ast.rs b/tremor-script/src/ast.rs index ef6516ffde..d98868d457 100644 --- a/tremor-script/src/ast.rs +++ b/tremor-script/src/ast.rs @@ -886,6 +886,8 @@ pub enum ImutExprInt<'script> { Merge(Box>), /// Path Path(Path<'script>), + /// A string literal + String(StringLit<'script>), /// Local - local variable Local { /// Local Index @@ -924,6 +926,27 @@ fn is_lit(e: &ImutExprInt) -> bool { matches!(e, ImutExprInt::Literal(_)) } +/// A string literal with interpolation +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct StringLit<'script> { + /// Id + pub mid: usize, + /// Elements + pub elements: StrLitElements<'script>, +} +impl_expr_mid!(StringLit); + +/// A part of a string literal with interpolation +#[derive(Clone, Debug, PartialEq, Serialize)] +pub enum StrLitElement<'script> { + /// A literal string + Lit(Cow<'script, str>), + /// An expression in a string interpolation + Expr(ImutExprInt<'script>), +} +/// we're forced to make this pub because of lalrpop +pub type StrLitElements<'script> = Vec>; + #[derive(Clone, Debug, PartialEq, Serialize)] /// Encapsulates an emit expression pub struct EmitExpr<'script> { diff --git a/tremor-script/src/ast/base_expr.rs b/tremor-script/src/ast/base_expr.rs index 19a3b57596..9356c519d5 100644 --- a/tremor-script/src/ast/base_expr.rs +++ b/tremor-script/src/ast/base_expr.rs @@ -137,6 +137,7 @@ impl<'script> BaseExpr for ImutExprInt<'script> { ImutExprInt::Record(e) => e.s(meta), ImutExprInt::Unary(e) => e.s(meta), ImutExprInt::Bytes(e) => e.s(meta), + ImutExprInt::String(e) => e.s(meta), } } fn e(&self, meta: &NodeMetas) -> Location { @@ -161,6 +162,7 @@ impl<'script> BaseExpr for ImutExprInt<'script> { ImutExprInt::Record(e) => e.e(meta), ImutExprInt::Unary(e) => e.e(meta), ImutExprInt::Bytes(e) => e.e(meta), + ImutExprInt::String(e) => e.e(meta), } } fn mid(&self) -> usize { @@ -183,6 +185,7 @@ impl<'script> BaseExpr for ImutExprInt<'script> { ImutExprInt::Record(e) => e.mid(), ImutExprInt::Unary(e) => e.mid(), ImutExprInt::Bytes(e) => e.mid(), + ImutExprInt::String(e) => e.mid(), } } } diff --git a/tremor-script/src/ast/raw.rs b/tremor-script/src/ast/raw.rs index 6ad1c1dd17..0844a11a01 100644 --- a/tremor-script/src/ast/raw.rs +++ b/tremor-script/src/ast/raw.rs @@ -23,8 +23,8 @@ use super::{ ImutExpr, ImutExprInt, ImutMatch, ImutPredicateClause, Invocable, Invoke, InvokeAggr, InvokeAggrFn, List, Literal, LocalPath, Match, Merge, MetadataPath, ModDoc, NodeMetas, Patch, PatchOperation, Path, Pattern, PredicateClause, PredicatePattern, Predicates, Record, - RecordPattern, Recur, Script, Segment, StatePath, TestExpr, TuplePattern, UnaryExpr, - UnaryOpKind, Warning, + RecordPattern, Recur, Script, Segment, StatePath, StrLitElement, StringLit, TestExpr, + TuplePattern, UnaryExpr, UnaryOpKind, Warning, }; use super::{upable::Upable, BytesPart}; use crate::errors::{ @@ -460,39 +460,36 @@ impl<'script> Upable<'script> for LiteralRaw<'script> { pub struct StringLitRaw<'script> { pub(crate) start: Location, pub(crate) end: Location, - pub(crate) string: Cow<'script, str>, - pub(crate) exprs: ImutExprsRaw<'script>, + pub(crate) elements: StrLitElementsRaw<'script>, } -/// we're forced to make this pub because of lalrpop -pub struct StrLitElements<'script>( - pub(crate) Vec>, - pub(crate) ImutExprsRaw<'script>, -); - -impl<'script> From> for StringLitRaw<'script> { - fn from(mut es: StrLitElements<'script>) -> StringLitRaw<'script> { - es.0.reverse(); - es.1.reverse(); - - let string = if es.0.len() == 1 { - es.0.pop().unwrap_or_default() - } else { - let mut s = String::new(); - for e in es.0 { - s.push_str(&e); - } - s.into() - }; - StringLitRaw { - start: Location::default(), - end: Location::default(), - string, - exprs: es.1, - } +#[derive(Clone, Debug, PartialEq, Serialize)] +pub enum StrLitElementRaw<'script> { + Lit(Cow<'script, str>), + Expr(ImutExprRaw<'script>), +} + +impl<'script> From> for StrLitElementRaw<'script> { + fn from(e: ImutExprRaw<'script>) -> Self { + StrLitElementRaw::Expr(e) + } +} + +impl<'script> From> for StrLitElementRaw<'script> { + fn from(e: Cow<'script, str>) -> Self { + StrLitElementRaw::Lit(e) + } +} + +impl<'script> From<&'script str> for StrLitElementRaw<'script> { + fn from(e: &'script str) -> Self { + StrLitElementRaw::Lit(e.into()) } } +/// we're forced to make this pub because of lalrpop +pub type StrLitElementsRaw<'script> = Vec>; + pub(crate) fn reduce2<'script>( expr: ImutExprInt<'script>, helper: &Helper, @@ -973,6 +970,7 @@ pub enum ImutExprRaw<'script> { impl<'script> Upable<'script> for ImutExprRaw<'script> { type Target = ImutExprInt<'script>; + #[allow(clippy::too_many_lines)] fn up<'registry>(self, helper: &mut Helper<'script, 'registry>) -> Result { let was_leaf = helper.possible_leaf; helper.possible_leaf = false; @@ -988,26 +986,62 @@ impl<'script> Upable<'script> for ImutExprRaw<'script> { ImutExprInt::Unary(Box::new(u.up(helper)?)).try_reduce(helper)? } ImutExprRaw::String(mut s) => { - let lit = ImutExprRaw::Literal(LiteralRaw { - start: s.start, - end: s.end, - value: Value::from(s.string), - }); - if s.exprs.is_empty() { - lit.up(helper)? - } else { - let mut args = vec![lit]; - args.append(&mut s.exprs); - ImutExprRaw::Invoke(InvokeRaw { - start: s.start, - end: s.end, - module: vec!["core".into(), "string".into()], - fun: "format".into(), - args, - }) - .up(helper)? - .try_reduce(helper)? + s.elements.reverse(); + let mut new = Vec::with_capacity(s.elements.len()); + for e in s.elements { + let next = match e { + StrLitElementRaw::Expr(e) => { + let i = e.up(helper)?; + let i = i.try_reduce(helper)?; + match i { + ImutExprInt::Literal(l) => l.value.as_str().map_or_else( + || StrLitElement::Lit(l.value.encode().into()), + |s| StrLitElement::Lit(s.to_string().into()), + ), + _ => StrLitElement::Expr(i), + } + } + StrLitElementRaw::Lit(l) => StrLitElement::Lit(l), + }; + if let StrLitElement::Lit(next_lit) = next { + // We need this because otherwise we run into lifetime issues + #[allow(clippy::option_if_let_else)] + if let Some(prev) = new.pop() { + match prev { + StrLitElement::Lit(l) => { + let mut o = l.into_owned(); + o.push_str(&next_lit); + new.push(StrLitElement::Lit(o.into())) + } + prev @ StrLitElement::Expr(..) => { + new.push(prev); + new.push(StrLitElement::Lit(next_lit)) + } + } + } else { + new.push(StrLitElement::Lit(next_lit)) + } + } else { + new.push(next) + } + } + let mid = helper.add_meta(s.start, s.end); + if new.len() == 1 { + match new.pop() { + Some(StrLitElement::Lit(l)) => { + let value = Value::from(l); + return Ok(ImutExprInt::Literal(Literal { mid, value })); + } + Some(other) => new.push(other), + None => (), + } + } else if new.is_empty() { + return Ok(ImutExprInt::Literal(Literal { + mid, + value: Value::from(""), + })); } + ImutExprInt::String(StringLit { mid, elements: new }) } ImutExprRaw::Record(r) => ImutExprInt::Record(r.up(helper)?).try_reduce(helper)?, ImutExprRaw::List(l) => ImutExprInt::List(l.up(helper)?).try_reduce(helper)?, diff --git a/tremor-script/src/errors.rs b/tremor-script/src/errors.rs index 17a3af48b2..ef2e7a5ed2 100644 --- a/tremor-script/src/errors.rs +++ b/tremor-script/src/errors.rs @@ -313,7 +313,7 @@ impl ErrorKind { } } UnterminatedInterpolation(_, _, _) => { - Some("Did you mean to write a literal '{' ? Escape it as '\\{' or '{{'.".to_string()) + Some("Did you mean to write a literal '#{' ? Escape it as '\\#{'.".to_string()) } BadAccessInLocal(_, _, key, _) if key == "nil" => { Some("Did you mean null?".to_owned()) diff --git a/tremor-script/src/grammar.lalrpop b/tremor-script/src/grammar.lalrpop index 38c43a3ec9..36cceb5007 100644 --- a/tremor-script/src/grammar.lalrpop +++ b/tremor-script/src/grammar.lalrpop @@ -642,59 +642,52 @@ Float: Value<'input> = { } StringLiteral: StringLitRaw<'input> = { - "heredoc" "heredoc" => { - let mut sl: StringLitRaw = es.into(); - sl.start = start; - sl.end = end; - sl + "heredoc" "heredoc" => { + StringLitRaw { + start, + end, + elements + } }, - "\"" "\"" => { - let mut sl: StringLitRaw = es.into(); - sl.start = start; - sl.end = end; - sl + "\"" "\"" => { + StringLitRaw { + start, + end, + elements + } }, "\"" "\"" => { StringLitRaw { start, end, - string: "".into(), - exprs: vec![], + elements: vec!["".into()], } }, } -StrLitElements: StrLitElements<'input> = { +StrLitElements: StrLitElementsRaw<'input> = { => { let mut es = es; - es.0.push(literal); - es - }, - - "\\{" => { - let mut es = es; - es.0.push("\\{".into()); + es.push(literal.into()); es }, - "\\}" => { + "\\#" => { let mut es = es; - es.0.push("\\}".into()); + es.push("#".into()); es }, - "{" "}" => { + "#{" "}" => { let mut es = es; - es.0.push("{}".into()); - es.1.push(expr); + es.push(expr.into()); es }, - => StrLitElements(vec![literal], vec![]), - "\\}" => StrLitElements(vec!["\\}".into()], vec![]), - "\\{" => StrLitElements(vec!["\\{".into()], vec![]), + => vec![literal.into()], + "\\#" => vec!["#".into()], - "{" "}" => StrLitElements(vec!["{}".into()], vec![expr]), + "#{" "}" => vec![expr.into()], } StringPart: Cow<'input, str> = { @@ -1347,8 +1340,7 @@ extern { "mod" => Token::Module, "." => Token::Dot, "\"" => Token::DQuote, - "\\{" => Token::QuotedSquigglyOpen, - "\\}" => Token::QuotedSquigglyClose, + "\\#" => Token::EscapedHash, ";" => Token::Semi, "::" => Token::ColonColon, ":" => Token::Colon, @@ -1357,6 +1349,7 @@ extern { "_" => Token::DontCare, "==" => Token::EqEq, "!=" => Token::NotEq, + "#{" => Token::Interpol, "{" => Token::LBrace, "}" => Token::RBrace, "(" => Token::LParen, diff --git a/tremor-script/src/interpreter/imut_expr.rs b/tremor-script/src/interpreter/imut_expr.rs index f73371d774..8a4e38d5c8 100644 --- a/tremor-script/src/interpreter/imut_expr.rs +++ b/tremor-script/src/interpreter/imut_expr.rs @@ -17,7 +17,6 @@ use super::{ test_predicate_expr, AggrType, Env, ExecOpts, LocalStack, FALSE, TRUE, }; -use crate::interpreter::value_to_index; use crate::prelude::*; use crate::registry::{Registry, TremorAggrFnWrapper, RECUR_REF}; use crate::stry; @@ -28,6 +27,7 @@ use crate::{ error_no_clause_hit, error_oops, Result, }, }; +use crate::{ast::StringLit, interpreter::value_to_index}; use crate::{ ast::{ BaseExpr, BinExpr, ImutComprehension, ImutExpr, ImutExprInt, ImutMatch, Invoke, InvokeAggr, @@ -120,6 +120,25 @@ where local: &'run LocalStack<'event>, ) -> Result>> { match self { + ImutExprInt::String(StringLit { elements, .. }) => { + // FIXME: optimize + let mut out = String::new(); + + for e in elements { + match e { + crate::ast::StrLitElement::Lit(l) => out.push_str(l), + crate::ast::StrLitElement::Expr(e) => { + let r = e.run(opts, env, event, state, meta, local)?; + if let Some(s) = r.as_str() { + out.push_str(&s); + } else { + out.push_str(r.encode().as_str()); + }; + } + } + } + Ok(Cow::Owned(out.into())) + } ImutExprInt::Recur(Recur { exprs, argc, .. }) => { #[allow(mutable_transmutes, clippy::transmute_ptr_to_ptr)] let local: &'run mut LocalStack<'event> = unsafe { mem::transmute(local) }; diff --git a/tremor-script/src/lexer.rs b/tremor-script/src/lexer.rs index 373af46f53..340f58ed3e 100644 --- a/tremor-script/src/lexer.rs +++ b/tremor-script/src/lexer.rs @@ -23,13 +23,13 @@ pub use crate::pos::*; use crate::Value; use beef::Cow; use simd_json::Writable; -use std::collections::VecDeque; use std::ffi::OsStr; use std::fmt; use std::io::Read; use std::iter::Peekable; use std::path::{Path, PathBuf}; use std::str::Chars; +use std::{collections::VecDeque, mem}; use tremor_common::file; use unicode_xid::UnicodeXID; @@ -106,7 +106,6 @@ pub enum Token<'input> { Dollar, /// a `.` Dot, - /// null Nil, /// a boolean @@ -124,10 +123,6 @@ pub enum Token<'input> { DQuote, /// a string literal StringLiteral(Cow<'input, str>), - /// A quoted `\{` or `{{` - QuotedSquigglyOpen, - /// A quoted `\}` or `{{` - QuotedSquigglyClose, // Keywords /// the `let` keyword @@ -184,7 +179,7 @@ pub enum Token<'input> { Module, /// the `_` token DontCare, - /// the `recure` token + /// the `recur` token Recur, // Symbols @@ -262,6 +257,10 @@ pub enum Token<'input> { RParen, /// Left brace `{` LBrace, + /// Interpolation Start `#{` + Interpol, + /// Escaped Hash `\#` + EscapedHash, /// Left pattern Bracket `%{` LPatBrace, /// Right Brace `}` @@ -527,10 +526,10 @@ impl<'input> fmt::Display for Token<'input> { Token::IntLiteral(value) => write!(f, "{}", value), Token::FloatLiteral(_, txt) => write!(f, "{}", txt), Token::DQuote => write!(f, "\""), - Token::QuotedSquigglyOpen => write!(f, "\\{{"), - Token::QuotedSquigglyClose => write!(f, "\\}}"), + Token::Interpol => write!(f, "#{{"), + Token::EscapedHash => write!(f, "\\#"), Token::StringLiteral(value) => { - // We do thos to ensure proper escaping + // We do those to ensure proper escaping let value: &str = &value; let s = Value::from(value).encode(); // Strip the quotes @@ -1621,8 +1620,9 @@ impl<'input> Lexer<'input> { Some((e, '\'')) => Ok((e, '\'')), Some((e, '"')) => Ok((e, '"')), Some((e, '\\')) => Ok((e, '\\')), - Some((e, '{')) => Ok((e, '{')), - Some((e, '}')) => Ok((e, '}')), + // Some((e, '{')) => Ok((e, '{')), + // Some((e, '}')) => Ok((e, '}')), + Some((e, '#')) => Ok((e, '#')), Some((e, '/')) => Ok((e, '/')), Some((e, 'b')) => Ok((e, '\u{8}')), // Backspace Some((e, 'f')) => Ok((e, '\u{c}')), // Form Feed @@ -1824,300 +1824,270 @@ impl<'input> Lexer<'input> { } } - #[allow(clippy::too_many_lines)] - /// Handle heredoc strings `"""` ... - fn hd( + #[allow(clippy::too_many_lines, clippy::clippy::too_many_arguments)] + fn handle_interpol( &mut self, - heredoc_start: Location, - mut segment_start: Location, - mut end: Location, - mut has_escapes: bool, - _string: &str, - mut res: Vec>, - ) -> Result>> { - // TODO: deduplicate by encapsulating all state in a struct and have some common operations on it - let mut heredoc_content = String::new(); + is_hd: bool, + error_prefix: &str, + has_escapes: &mut bool, + total_start: Location, + end_inner: Location, + segment_start: &mut Location, + end: &mut Location, + res: &mut Vec>, + content: &mut String, + ) -> Result<()> { + end.shift('{'); + self.bump(); + if !content.is_empty() { + let mut c = String::new(); + mem::swap(&mut c, content); + let token = if *has_escapes { + // The string was modified so we can't use the slice + Token::StringLiteral(c.into()) + } else { + Token::StringLiteral( + self.slice(*segment_start, end_inner) + .map_or_else(|| Cow::from(c), Cow::from), + ) + }; + res.push(self.spanned2(*segment_start, end_inner, token)); + *has_escapes = false; + } + *segment_start = end_inner; + res.push(self.spanned2(*segment_start, *end, Token::Interpol)); + let mut pcount = 0; loop { - let ch = self.bump(); - match ch { - Some((e, '\n')) => { - end = e; - heredoc_content.push('\n'); - res.push(self.spanned2( - segment_start, - end, - Token::StringLiteral(heredoc_content.into()), - )); - end.shift('\n'); - segment_start = end; - heredoc_content = String::new(); - } - Some((end_inner, '\\')) => { - let (mut e, c) = self.escape_code(&heredoc_start, end_inner)?; - if c == '{' || c == '}' { - if !heredoc_content.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(heredoc_content.into()) - } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(heredoc_content.into()), - |slice| Token::StringLiteral(slice.into()), - ) - }; - res.push(self.spanned2(segment_start, end_inner, token)); - heredoc_content = String::new(); - has_escapes = false; + match self.next() { + Some(Ok(s)) => { + match &s.value { + Token::RBrace if pcount == 0 => { + *segment_start = s.span.pp_end; + res.push(s); + break; } - if c == '{' { - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyOpen)); - } else { - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyClose)); - }; - e.shift(c); - segment_start = e; - end = e; - } else { - has_escapes = true; - heredoc_content.push(c); - end = e; - end.shift(c); - } + Token::RBrace => { + pcount -= 1; + } + Token::LBrace | Token::Interpol => { + pcount += 1; + } + _ => {} + }; + res.push(s); } - Some((end_inner, '}')) => { - if let Some((mut e, '}')) = self.lookahead() { - self.bump(); - // TODO: deduplicate - if !heredoc_content.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(heredoc_content.into()) - } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(heredoc_content.into()), - |slice| Token::StringLiteral(slice.into()), + // intercept error and extend the token to match this outer heredoc + // with interpolation + // otherwise we will not get the whole heredoc in error messages + Some(Err(error)) => { + let end_location = error.context().1.map_or_else( + || res.last().map_or(*end, |last| last.span.end), + |inner_error_range| inner_error_range.1, + ); + + let Error(kind, ..) = error; + + let token_str = self + .slice_full_lines(&total_start, &end_location) + .unwrap_or_else(|| format!("{}{}", error_prefix, content)); + let mut end = total_start; + end.shift_str(&token_str); + let unfinished_token = + UnfinishedToken::new(Range::from((total_start, end_location)), token_str); + let error = match kind { + ErrorKind::UnterminatedExtractor(outer, location, _) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + ErrorKind::UnterminatedExtractor(outer, location, unfinished_token) + } + ErrorKind::UnterminatedIdentLiteral(outer, location, _) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + ErrorKind::UnterminatedIdentLiteral(outer, location, unfinished_token) + } + ErrorKind::UnterminatedHereDoc(outer, location, _) + | ErrorKind::TailingHereDoc(outer, location, _, _) => { + if is_hd { + // unterminated heredocs within interpolation are better reported + // as unterminated interpolation + ErrorKind::UnterminatedInterpolation( + Range::from((total_start, end.move_down_lines(2))), + Range::from((total_start, end)), + unfinished_token, ) - }; - res.push(self.spanned2(segment_start, end_inner, token)); - heredoc_content = String::new(); - has_escapes = false; + } else { + let outer = outer.expand_lines( + outer.0.line().saturating_sub(total_start.line()), + ); + + ErrorKind::UnterminatedHereDoc(outer, location, unfinished_token) + } } - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyClose)); - e.shift('}'); - segment_start = e; - end = e; - continue; - } else { - heredoc_content.push('}'); - end = end_inner; - end.shift('}'); - } - } - Some((end_inner, '{')) => { - if let Some((e, '}')) = self.lookahead() { - self.bump(); - heredoc_content.push('{'); - heredoc_content.push('}'); - end = e; - end.shift('}'); - continue; - } else if let Some((mut e, '{')) = self.lookahead() { - self.bump(); - // TODO: deduplicate - if !heredoc_content.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(heredoc_content.into()) + ErrorKind::UnterminatedInterpolation(outer, location, _) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + ErrorKind::UnterminatedInterpolation(outer, location, unfinished_token) + } + ErrorKind::UnexpectedEscapeCode(outer, location, _, found) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + ErrorKind::UnexpectedEscapeCode( + outer, + location, + unfinished_token, + found, + ) + } + ErrorKind::UnterminatedStringLiteral(outer, location, _) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + if is_hd { + ErrorKind::UnterminatedStringLiteral( + outer, + location, + unfinished_token, + ) } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(heredoc_content.into()), - |slice| Token::StringLiteral(slice.into()), + let mut toekn_end = *segment_start; + toekn_end.shift('#'); + toekn_end.shift('{'); + ErrorKind::UnterminatedInterpolation( + outer, + Range::from((*segment_start, toekn_end)), + unfinished_token, ) - }; - res.push(self.spanned2(segment_start, end_inner, token)); - heredoc_content = String::new(); - has_escapes = false; + } } - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyOpen)); - e.shift('{'); - segment_start = e; - end = e; - continue; - } - if !heredoc_content.is_empty() { - let token = if has_escapes { + ErrorKind::InvalidUTF8Sequence(outer, location, _) => { + // expand to start line of heredoc, so we get a proper context + let outer = outer + .expand_lines(outer.0.line().saturating_sub(total_start.line())); + ErrorKind::InvalidUTF8Sequence(outer, location, unfinished_token) + } + e => e, + }; + return Err(error.into()); + } + None => { + let end_location = self.chars.current(); + let token_str = self + .slice_full_lines(&total_start, end_location) + .unwrap_or_else(|| format!("{}{}", error_prefix, content)); + return Err(ErrorKind::UnterminatedInterpolation( + Range::from((total_start, end.move_down_lines(2))), + Range::from((*segment_start, *end)), + UnfinishedToken::new(Range::from((total_start, *end_location)), token_str), + ) + .into()); + } + } + } + Ok(()) + } + + #[allow(clippy::clippy::too_many_arguments)] + fn handle_qs_hd_generic( + &mut self, + is_hd: bool, + error_prefix: &str, + has_escapes: &mut bool, + total_start: Location, + segment_start: &mut Location, + end: &mut Location, + res: &mut Vec>, + content: &mut String, + lc: (Location, char), + ) -> Result<()> { + match lc { + (end_inner, '\\') => { + let (mut e, c) = self.escape_code(&total_start, end_inner)?; + if c == '#' { + if !content.is_empty() { + let mut c = String::new(); + mem::swap(&mut c, content); + let token = if *has_escapes { // The string was modified so we can't use the slice - Token::StringLiteral(heredoc_content.into()) + Token::StringLiteral(c.into()) } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(heredoc_content.into()), + self.slice(*segment_start, end_inner).map_or_else( + || Token::StringLiteral(c.into()), |slice| Token::StringLiteral(slice.into()), ) }; - res.push(self.spanned2(segment_start, end_inner, token)); - heredoc_content = String::new(); - has_escapes = false; - } - segment_start = end_inner; - end = end_inner; - end.shift('{'); - res.push(self.spanned2(segment_start, end, Token::LBrace)); - let mut pcount = 0; - loop { - match self.next() { - Some(Ok(s)) => { - match &s.value { - Token::RBrace if pcount == 0 => { - segment_start = s.span.pp_end; - res.push(s); - break; - } - Token::RBrace => { - pcount -= 1; - } - Token::LBrace => { - pcount += 1; - } - _ => {} - }; - res.push(s); - } - // intercept error and extend the token to match this outer heredoc - // with interpolation - // otherwise we will not get the whole heredoc in error messages - Some(Err(error)) => { - let end_location = error.context().1.map_or_else( - || res.last().map_or(end, |last| last.span.end), - |inner_error_range| inner_error_range.1, - ); - - let Error(kind, ..) = error; - - let token_str = self - .slice_full_lines(&heredoc_start, &end_location) - .unwrap_or_else(|| format!("\"\"\"\n{}", heredoc_content)); - let unfinished_token = UnfinishedToken::new( - Range::from((heredoc_start, end_location)), - token_str, - ); - let error = match kind { - ErrorKind::UnterminatedExtractor(outer, location, _) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::UnterminatedExtractor( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnterminatedIdentLiteral(outer, location, _) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::UnterminatedIdentLiteral( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnterminatedHereDoc(_, _, _) - | ErrorKind::TailingHereDoc(_, _, _, _) => { - // unterminated heredocs within interpolation are better reported - // as unterminated interpolation - ErrorKind::UnterminatedInterpolation( - Range::from((heredoc_start, end.move_down_lines(2))), - Range::from((heredoc_start, end)), - unfinished_token, - ) - } - ErrorKind::UnterminatedInterpolation(outer, location, _) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::UnterminatedInterpolation( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnexpectedEscapeCode(outer, location, _, found) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::UnexpectedEscapeCode( - outer, - location, - unfinished_token, - found, - ) - } - ErrorKind::UnterminatedStringLiteral(outer, location, _) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::UnterminatedStringLiteral( - outer, - location, - unfinished_token, - ) - } - ErrorKind::InvalidUTF8Sequence(outer, location, _) => { - // expand to start line of heredoc, so we get a proper context - let outer = outer.expand_lines( - outer.0.line().saturating_sub(heredoc_start.line()), - ); - ErrorKind::InvalidUTF8Sequence( - outer, - location, - unfinished_token, - ) - } - e => e, - }; - return Err(error.into()); - } - None => { - let end_location = self.chars.current(); - let token_str = self - .slice_full_lines(&heredoc_start, end_location) - .unwrap_or_else(|| format!("\"\"\"\n{}", heredoc_content)); - return Err(ErrorKind::UnterminatedInterpolation( - Range::from((heredoc_start, end.move_down_lines(2))), - Range::from((segment_start, end)), - UnfinishedToken::new( - Range::from((heredoc_start, *end_location)), - token_str, - ), - ) - .into()); - } - } + res.push(self.spanned2(*segment_start, end_inner, token)); + *has_escapes = false; } + res.push(self.spanned2(end_inner, e, Token::EscapedHash)); + e.shift(c); + *segment_start = e; + *end = e; + } else { + *has_escapes = true; + content.push(c); + *end = e; + } + } + (end_inner, '#') => { + if let Some((e, '{')) = self.lookahead() { + *end = e; + self.handle_interpol( + is_hd, + error_prefix, + has_escapes, + total_start, + end_inner, + segment_start, + end, + res, + content, + )?; + } else { + content.push('#'); + *end = end_inner; } + } + (e, other) => { + content.push(other); + *end = e; + end.shift(other); + } + } + Ok(()) + } + + /// Handle heredoc strings `"""` ... + fn hd( + &mut self, + heredoc_start: Location, + mut segment_start: Location, + mut end: Location, + mut has_escapes: bool, + _string: &str, + mut res: Vec>, + ) -> Result>> { + // TODO: deduplicate by encapsulating all state in a struct and have some common operations on it + let mut content = String::new(); + loop { + match self.bump() { Some((e, '"')) => { // If the current line is just a `"""` then we are at the end of the heredoc - res.push(self.spanned2( - segment_start, - e, - Token::StringLiteral(heredoc_content.into()), - )); + res.push(self.spanned2(segment_start, e, Token::StringLiteral(content.into()))); segment_start = e; - heredoc_content = String::new(); - heredoc_content.push('"'); + content = String::new(); + content.push('"'); end = if let Some((e, '"')) = self.lookahead() { self.bump(); - heredoc_content.push('"'); + content.push('"'); if let Some((e, '"')) = self.lookahead() { self.bump(); let mut heredoc_end = e; heredoc_end.shift('"'); - res.push(self.spanned2(segment_start, heredoc_end, Token::HereDoc)); // (0, vec![]))); + res.push(self.spanned2(segment_start, heredoc_end, Token::HereDoc)); return Ok(res); } else { e @@ -2127,18 +2097,36 @@ impl<'input> Lexer<'input> { }; end.shift('"'); } - Some((e, other)) => { - heredoc_content.push(other as char); + Some((e, '\n')) => { end = e; - end.shift(other); + content.push('\n'); + res.push(self.spanned2( + segment_start, + end, + Token::StringLiteral(content.into()), + )); + end.shift('\n'); + segment_start = end; + content = String::new(); + } + Some(lc) => { + self.handle_qs_hd_generic( + true, + "\"\"\"\n", + &mut has_escapes, + heredoc_start, + &mut segment_start, + &mut end, + &mut res, + &mut content, + lc, + )?; } - None => { // We reached EOF - let token_str = self.slice_until_eof(&heredoc_start).map_or_else( - || format!(r#""""\n{}"#, heredoc_content), - ToString::to_string, - ); + let token_str = self + .slice_until_eof(&heredoc_start) + .map_or_else(|| format!(r#""""\n{}"#, content), ToString::to_string); let range = Range::from((heredoc_start, end)); return Err(ErrorKind::UnterminatedHereDoc( range.expand_lines(2), @@ -2155,7 +2143,7 @@ impl<'input> Lexer<'input> { /// Handle quote strings `"` ... fn qs( &mut self, - string_start: Location, + total_start: Location, mut segment_start: Location, mut end: Location, mut has_escapes: bool, @@ -2164,37 +2152,6 @@ impl<'input> Lexer<'input> { ) -> Result>> { loop { match self.bump() { - Some((end_inner, '\\')) => { - let (mut e, c) = self.escape_code(&string_start, end_inner)?; - if c == '{' || c == '}' { - if !string.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(string.into()) - } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(string.into()), - |slice| Token::StringLiteral(slice.into()), - ) - }; - res.push(self.spanned2(segment_start, end_inner, token)); - string = String::new(); - has_escapes = false; - } - if c == '{' { - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyOpen)); - } else { - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyClose)); - }; - e.shift(c); - segment_start = e; - end = e; - } else { - has_escapes = true; - string.push(c); - end = e; - } - } Some((mut end, '"')) => { // If the string is empty we kind of don't need it. if !string.is_empty() { @@ -2214,224 +2171,45 @@ impl<'input> Lexer<'input> { res.push(self.spanned2(start, end, Token::DQuote)); return Ok(res); } - Some((end_inner, '}')) => { - if let Some((mut e, '}')) = self.lookahead() { - self.bump(); - // TODO: deduplicate - if !string.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(string.into()) - } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(string.into()), - |slice| Token::StringLiteral(slice.into()), - ) - }; - res.push(self.spanned2(segment_start, end, token)); - string = String::new(); - has_escapes = false; - } - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyClose)); - e.shift('}'); - segment_start = e; - end = e; - } else { - string.push('}'); - end = end_inner; - end.shift('}'); - } - } - Some((end_inner, '{')) => { - if let Some((e, '}')) = self.lookahead() { - self.bump(); - string.push('{'); - string.push('}'); - end = e; - continue; - } else if let Some((mut e, '{')) = self.lookahead() { - self.bump(); - // TODO: deduplicate - if !string.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(string.into()) - } else { - self.slice(segment_start, end_inner).map_or_else( - || Token::StringLiteral(string.into()), - |slice| Token::StringLiteral(slice.into()), - ) - }; - res.push(self.spanned2(segment_start, end, token)); - string = String::new(); - has_escapes = false; - } - - res.push(self.spanned2(end_inner, e, Token::QuotedSquigglyOpen)); - e.shift('{'); - segment_start = e; - end = e; - continue; - } - if !string.is_empty() { - let token = if has_escapes { - // The string was modified so we can't use the slice - Token::StringLiteral(string.into()) - } else { - Token::StringLiteral( - self.slice(segment_start, end_inner) - .map_or_else(|| Cow::from(string), Cow::from), - ) - }; - res.push(self.spanned2(segment_start, end_inner, token)); - string = String::new(); - has_escapes = false; - } - segment_start = end_inner; - end = end_inner; - end.shift('{'); - res.push(self.spanned2(segment_start, end, Token::LBrace)); - let mut pcount = 0; - loop { - match self.next() { - Some(Ok(s)) => { - match &s.value { - Token::RBrace if pcount == 0 => { - segment_start = s.span.pp_end; - res.push(s); - break; - } - Token::RBrace => { - pcount -= 1; - } - Token::LBrace => { - pcount += 1; - } - _ => {} - }; - res.push(s); - } - // intercept error and extend the token to match this outer string - // with interpolation - // otherwise we will not get the whole string in error messages - Some(Err(Error(kind, _))) => { - let token_str = self - .slice_until_eol(&string_start) - .map_or_else(|| format!("\"{}", string), ToString::to_string); - let mut end = string_start; - end.shift_str(&token_str); - let unfinished_token = UnfinishedToken::new( - Range::from((string_start, end)), - token_str, - ); - let error = match kind { - ErrorKind::UnterminatedExtractor(outer, location, _) => { - ErrorKind::UnterminatedExtractor( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnterminatedIdentLiteral(outer, location, _) => { - ErrorKind::UnterminatedIdentLiteral( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnterminatedStringLiteral(_, _, _) => { - // unterminated strings within interpolation are better reported - // as unterminated interpolation - ErrorKind::UnterminatedInterpolation( - Range::from((segment_start, end)).expand_lines(2), - Range::from((segment_start, segment_start)), - unfinished_token, - ) - } - ErrorKind::UnterminatedInterpolation(outer, location, _) => { - ErrorKind::UnterminatedInterpolation( - outer, - location, - unfinished_token, - ) - } - ErrorKind::UnexpectedEscapeCode(outer, location, _, found) => { - ErrorKind::UnexpectedEscapeCode( - outer, - location, - unfinished_token, - found, - ) - } - ErrorKind::UnterminatedHereDoc(outer, location, _) => { - ErrorKind::UnterminatedHereDoc( - outer, - location, - unfinished_token, - ) - } - ErrorKind::InvalidUTF8Sequence(outer, location, _) => { - ErrorKind::InvalidUTF8Sequence( - outer, - location, - unfinished_token, - ) - } - e => e, - }; - return Err(error.into()); - } - None => { - let token_str = self - .slice_until_eol(&string_start) - .map_or_else(|| format!("\"{}", string), ToString::to_string); - let mut token_end = string_start; - token_end.shift_str(&token_str); - let range = Range::from((segment_start, end)); - return Err(ErrorKind::UnterminatedInterpolation( - range.expand_lines(2), - range, - UnfinishedToken::new( - Range::from((string_start, token_end)), - token_str, - ), - ) - .into()); - } - } - } - } Some((end, '\n')) => { let token_str = self - .slice_until_eol(&string_start) + .slice_until_eol(&total_start) .map_or_else(|| format!("\"{}", string), ToString::to_string); - let mut token_end = string_start; + let mut token_end = total_start; token_end.shift_str(&token_str); - let range = Range::from((string_start, end)); + let range = Range::from((total_start, end)); return Err(ErrorKind::UnterminatedStringLiteral( range.expand_lines(2), range, - UnfinishedToken::new(Range::from((string_start, token_end)), token_str), + UnfinishedToken::new(Range::from((total_start, token_end)), token_str), ) .into()); } - Some((e, other)) => { - string.push(other as char); - end = e; - end.shift(other); + Some(lc) => { + self.handle_qs_hd_generic( + false, + "\"", + &mut has_escapes, + total_start, + &mut segment_start, + &mut end, + &mut res, + &mut string, + lc, + )?; } None => { let token_str = self - .slice_until_eol(&string_start) + .slice_until_eol(&total_start) .map_or_else(|| format!("\"{}", string), ToString::to_string); - let mut token_end = string_start; + let mut token_end = total_start; token_end.shift_str(&token_str); - let range = Range::from((string_start, end)); + let range = Range::from((total_start, end)); return Err(ErrorKind::UnterminatedStringLiteral( range.expand_lines(2), range, - UnfinishedToken::new(Range::from((string_start, token_end)), token_str), + UnfinishedToken::new(Range::from((total_start, token_end)), token_str), ) .into()); } @@ -2871,38 +2649,39 @@ mod tests { r#" ~ "# => Token::DQuote, }; lex_ok! { - r#" "hello {7}" "#, + r#" "hello #{7}" "#, r#" ~ "# => Token::DQuote, r#" ~~~~~~ "# => Token::StringLiteral("hello ".into()), - r#" ~ "# => Token::LBrace, - r#" ~ "# => Token::IntLiteral(7), - r#" ~ "# => Token::RBrace, - r#" ~ "# => Token::DQuote, + r#" ~~ "# => Token::Interpol, + r#" ~ "# => Token::IntLiteral(7), + r#" ~ "# => Token::RBrace, + r#" ~ "# => Token::DQuote, }; + // We can't use `r#""#` for the string since we got a a `"#` in it lex_ok! { - r#" "{7} hello" "#, + " \"#{7} hello\" ", r#" ~ "# => Token::DQuote, - r#" ~ "# => Token::LBrace, - r#" ~ "# => Token::IntLiteral(7), - r#" ~ "# => Token::RBrace, - r#" ~~~~~~ "# => Token::StringLiteral(" hello".into()), - r#" ~ "# => Token::DQuote, + r#" ~~ "# => Token::Interpol, + r#" ~ "# => Token::IntLiteral(7), + r#" ~ "# => Token::RBrace, + r#" ~~~~~~ "# => Token::StringLiteral(" hello".into()), + r#" ~ "# => Token::DQuote, }; lex_ok! { - r#" "hello { "snot {7}" }" "#, + r#" "hello #{ "snot #{7}" }" "#, r#" ~ "# => Token::DQuote, r#" ~~~~~~ "# => Token::StringLiteral("hello ".into()), - r#" ~ "# => Token::LBrace, + r#" ~~ "# => Token::Interpol, r#" ~ "# => Token::DQuote, - r#" ~~~~ "# => Token::StringLiteral("snot ".into()), - r#" ~ "# => Token::LBrace, - r#" ~ "# => Token::IntLiteral(7), - r#" ~ "# => Token::RBrace, - r#" ~ "# => Token::DQuote, - r#" ~ "# => Token::RBrace, - r#" ~ "# => Token::DQuote, + r#" ~~~~~ "# => Token::StringLiteral("snot ".into()), + r#" ~~ "# => Token::Interpol, + r#" ~ "# => Token::IntLiteral(7), + r#" ~ "# => Token::RBrace, + r#" ~ "# => Token::DQuote, + r#" ~ "# => Token::RBrace, + r#" ~ "# => Token::DQuote, }; Ok(()) } @@ -3049,28 +2828,16 @@ mod tests { r#" ~ "# => Token::DQuote, }; lex_ok! { - r#" "\{" "#, - r#" ~ "# => Token::DQuote, - r#" ~~ "# => Token::QuotedSquigglyOpen, - r#" ~ "# => Token::DQuote, - }; - lex_ok! { - r#" "{{" "#, - r#" ~ "# => Token::DQuote, - r#" ~~ "# => Token::QuotedSquigglyOpen, - r#" ~ "# => Token::DQuote, - }; - lex_ok! { - r#" "\}" "#, + r#" "{" "#, r#" ~ "# => Token::DQuote, - r#" ~~ "# => Token::QuotedSquigglyClose, - r#" ~ "# => Token::DQuote, + r#" ~ "# => Token::StringLiteral("{".into()), + r#" ~ "# => Token::DQuote, }; lex_ok! { - r#" "}}" "#, + r#" "}" "#, r#" ~ "# => Token::DQuote, - r#" ~~ "# => Token::QuotedSquigglyClose, - r#" ~ "# => Token::DQuote, + r#" ~ "# => Token::StringLiteral("}".into()), + r#" ~ "# => Token::DQuote, }; lex_ok! { r#" "{}" "#, @@ -3088,8 +2855,7 @@ mod tests { lex_ok! { r#" "a\nb}}" "#, r#" ~ "# => Token::DQuote, - r#" ~~~~ "# => Token::StringLiteral("a\nb".into()), - r#" ~~ "# => Token::QuotedSquigglyClose, + r#" ~~~~~~ "# => Token::StringLiteral("a\nb}}".into()), r#" ~ "# => Token::DQuote, }; From 637ac0a5c0ce82b27bd84e5ab639cc929bc6b8d6 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Tue, 2 Feb 2021 18:05:37 +0100 Subject: [PATCH 4/9] Fix CI update Signed-off-by: Heinz N. Gies --- .github/workflows/checks.yaml | 2 +- .github/workflows/eqc.yaml | 6 +++--- .github/workflows/integration.yaml | 2 +- .github/workflows/tests.yml | 4 ++-- tremor-script/src/interpreter/imut_expr.rs | 4 +--- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 8b9ab84f20..28261b867c 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -60,7 +60,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Install deps - run: sudo apt-get -qy update && sudo apt-get install -y libssl1.0-dev libssl1.0 + run: sudo apt-get -qy update && sudo apt-get install -y libssl-dev libssl1.1 - uses: actions-rs/toolchain@v1 with: override: true diff --git a/.github/workflows/eqc.yaml b/.github/workflows/eqc.yaml index 275c78d3a4..bfccad8a39 100644 --- a/.github/workflows/eqc.yaml +++ b/.github/workflows/eqc.yaml @@ -7,11 +7,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1.0.0 - - uses: gleam-lang/setup-erlang@v1.0.0 + - uses: gleam-lang/setup-erlang@v1.1.2 with: - otp-version: 22.1 + otp-version: 23.1 - name: Install deps - run: sudo apt-get -qy update && sudo apt-get install -y libssl1.0-dev libssl1.0 + run: sudo apt-get -qy update && sudo apt-get install -y libssl-dev libssl1.1 - name: Install yaml2json run: go get github.com/bronze1man/yaml2json - name: install EQC diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 8943977471..629ab3922a 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v1 - name: Install deps if: matrix.os == 'ubuntu-latest' - run: sudo apt-get -qy update && sudo apt-get install -y libssl1.0-dev libssl1.0 + run: sudo apt-get -qy update && sudo apt-get install -y libssl-dev libssl1.1 - uses: actions-rs/toolchain@v1 with: override: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 18a53f4275..1a862774da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install deps [Linux] - run: sudo apt-get -qy update && sudo apt-get install -y libssl1.0-dev libssl1.0 + run: sudo apt-get -qy update && sudo apt-get install -y libssl-dev libssl1.1 - uses: actions-rs/toolchain@v1 with: override: true @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install deps - run: sudo apt-get -qy update && sudo apt-get install -y libssl1.0-dev libssl1.0 + run: sudo apt-get -qy update && sudo apt-get install -y libssl-dev libssl1.1 - uses: actions-rs/toolchain@v1 with: override: true diff --git a/tremor-script/src/interpreter/imut_expr.rs b/tremor-script/src/interpreter/imut_expr.rs index 8a4e38d5c8..debaa91e06 100644 --- a/tremor-script/src/interpreter/imut_expr.rs +++ b/tremor-script/src/interpreter/imut_expr.rs @@ -121,9 +121,7 @@ where ) -> Result>> { match self { ImutExprInt::String(StringLit { elements, .. }) => { - // FIXME: optimize - let mut out = String::new(); - + let mut out = String::with_capacity(128); for e in elements { match e { crate::ast::StrLitElement::Lit(l) => out.push_str(l), From b9a481b0163f731af4c32a7705f68b66f9939c5a Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Tue, 2 Feb 2021 19:03:37 +0100 Subject: [PATCH 5/9] Fix integration tests Signed-off-by: Heinz N. Gies --- tremor-cli/tests/integration/system_metrics/metrics.trickle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tremor-cli/tests/integration/system_metrics/metrics.trickle b/tremor-cli/tests/integration/system_metrics/metrics.trickle index 0c327a63c1..0c9494f0d1 100644 --- a/tremor-cli/tests/integration/system_metrics/metrics.trickle +++ b/tremor-cli/tests/integration/system_metrics/metrics.trickle @@ -13,7 +13,7 @@ script default => "out" end; - emit event => "{port}"; + emit event => "#{port}"; end; create script process; From bfbe61b4d15f4d131f1cc285c7d55283c8c0a14c Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Wed, 3 Feb 2021 11:00:00 +0100 Subject: [PATCH 6/9] Improve errors for #{} Signed-off-by: Heinz N. Gies --- .../lexer_heredoc_interpolation/error.txt | 2 +- .../lexer_heredoc_interpolation2/error.txt | 2 +- .../lexer_heredoc_interpolation3/error.txt | 2 +- .../lexer_string_interpolation/error.txt | 2 +- .../lexer_string_interpolation2/error.txt | 2 +- .../string_interpolation_eof/error.txt | 2 +- tremor-script/src/errors.rs | 49 +++++++++++-------- tremor-script/src/lexer.rs | 22 +++++++++ 8 files changed, 56 insertions(+), 27 deletions(-) diff --git a/tests/script_errors/lexer_heredoc_interpolation/error.txt b/tests/script_errors/lexer_heredoc_interpolation/error.txt index f910e7b25d..30e2417443 100644 --- a/tests/script_errors/lexer_heredoc_interpolation/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation/error.txt @@ -2,4 +2,4 @@ Error: 1 | """ 2 | #{ | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_heredoc_interpolation2/error.txt b/tests/script_errors/lexer_heredoc_interpolation2/error.txt index 9d422d6074..00b2ccd81d 100644 --- a/tests/script_errors/lexer_heredoc_interpolation2/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation2/error.txt @@ -2,4 +2,4 @@ Error: 1 | """ 2 | #{""" | ^^^^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_heredoc_interpolation3/error.txt b/tests/script_errors/lexer_heredoc_interpolation3/error.txt index 998c0aec51..32175e4bca 100644 --- a/tests/script_errors/lexer_heredoc_interpolation3/error.txt +++ b/tests/script_errors/lexer_heredoc_interpolation3/error.txt @@ -2,4 +2,4 @@ Error: 2 | let x = """ 3 | #{"""; | ^^^^^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_string_interpolation/error.txt b/tests/script_errors/lexer_string_interpolation/error.txt index 0df1d687ff..7ff4ecf160 100644 --- a/tests/script_errors/lexer_string_interpolation/error.txt +++ b/tests/script_errors/lexer_string_interpolation/error.txt @@ -1,4 +1,4 @@ Error: 1 | "#{" | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/lexer_string_interpolation2/error.txt b/tests/script_errors/lexer_string_interpolation2/error.txt index 6c9b9e7ac1..87481cd0a9 100644 --- a/tests/script_errors/lexer_string_interpolation2/error.txt +++ b/tests/script_errors/lexer_string_interpolation2/error.txt @@ -1,4 +1,4 @@ Error: 1 | let x = "before #{ { {}"; | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_eof/error.txt b/tests/script_errors/string_interpolation_eof/error.txt index d212cf6c17..9f55c83ee1 100644 --- a/tests/script_errors/string_interpolation_eof/error.txt +++ b/tests/script_errors/string_interpolation_eof/error.txt @@ -1,4 +1,4 @@ Error: 1 | let si = "before #{ | ^^ It looks like you forgot to terminate a string interpolation with a closing '}' - | NOTE: Did you mean to write a literal '#{' ? Escape it as '\#{'. \ No newline at end of file + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tremor-script/src/errors.rs b/tremor-script/src/errors.rs index ef2e7a5ed2..360dd024f7 100644 --- a/tremor-script/src/errors.rs +++ b/tremor-script/src/errors.rs @@ -174,19 +174,19 @@ impl ErrorKind { AccessError, AggrInAggr, ArrayOutOfRange, AssignIntoArray, AssignToConst, BadAccessInEvent, BadAccessInGlobal, BadAccessInLocal, BadAccessInState, BadArity, BadArrayIndex, BadType, BinaryDrop, BinaryEmit, CantSetArgsConst, CantSetGroupConst, - CantSetWindowConst, Common, DecreasingRange, DoubleConst, DoubleStream, EmptyScript, - ExtraToken, Generic, Grok, InvalidAssign, InvalidBinary, InvalidBitshift, InvalidConst, - InvalidDrop, InvalidEmit, InvalidExtractor, InvalidFloatLiteral, InvalidFn, - InvalidHexLiteral, InvalidInfluxData, InvalidIntLiteral, InvalidMod, InvalidRecur, - InvalidToken, InvalidUTF8Sequence, InvalidUnary, Io, JSONError, MergeTypeConflict, - MissingEffectors, MissingFunction, MissingModule, ModuleNotFound, Msg, NoClauseHit, - NoConstsAllowed, NoLocalsAllowed, NoObjectError, NotConstant, NotFound, Oops, - ParseIntError, ParserError, PatchKeyExists, PreprocessorError, QueryNodeDuplicateName, - QueryNodeReservedName, QueryStreamNotDefined, RuntimeError, TailingHereDoc, - TypeConflict, UnexpectedCharacter, UnexpectedEndOfStream, UnexpectedEscapeCode, - UnrecognizedToken, UnterminatedExtractor, UnterminatedHereDoc, - UnterminatedIdentLiteral, UnterminatedInterpolation, UnterminatedStringLiteral, - UpdateKeyMissing, Utf8Error, ValueError, + CantSetWindowConst, Common, DecreasingRange, DoubleConst, DoubleStream, + EmptyInterpolation, EmptyScript, ExtraToken, Generic, Grok, InvalidAssign, + InvalidBinary, InvalidBitshift, InvalidConst, InvalidDrop, InvalidEmit, + InvalidExtractor, InvalidFloatLiteral, InvalidFn, InvalidHexLiteral, InvalidInfluxData, + InvalidIntLiteral, InvalidMod, InvalidRecur, InvalidToken, InvalidUTF8Sequence, + InvalidUnary, Io, JSONError, MergeTypeConflict, MissingEffectors, MissingFunction, + MissingModule, ModuleNotFound, Msg, NoClauseHit, NoConstsAllowed, NoLocalsAllowed, + NoObjectError, NotConstant, NotFound, Oops, ParseIntError, ParserError, PatchKeyExists, + PreprocessorError, QueryNodeDuplicateName, QueryNodeReservedName, + QueryStreamNotDefined, RuntimeError, TailingHereDoc, TypeConflict, UnexpectedCharacter, + UnexpectedEndOfStream, UnexpectedEscapeCode, UnrecognizedToken, UnterminatedExtractor, + UnterminatedHereDoc, UnterminatedIdentLiteral, UnterminatedInterpolation, + UnterminatedStringLiteral, UpdateKeyMissing, Utf8Error, ValueError, }; match self { NoClauseHit(outer) @@ -245,6 +245,7 @@ impl ErrorKind { | UpdateKeyMissing(outer, inner, _) | UnterminatedHereDoc(outer, inner, _) | UnterminatedInterpolation(outer, inner, _) + | EmptyInterpolation(outer, inner, _) | TailingHereDoc(outer, inner, _, _) | Generic(outer, inner, _) | AggrInAggr(outer, inner) @@ -274,15 +275,16 @@ impl ErrorKind { } pub(crate) fn token(&self) -> Option { use ErrorKind::{ - InvalidFloatLiteral, InvalidHexLiteral, InvalidIntLiteral, InvalidUTF8Sequence, - TailingHereDoc, UnexpectedCharacter, UnexpectedEscapeCode, UnterminatedExtractor, - UnterminatedHereDoc, UnterminatedIdentLiteral, UnterminatedInterpolation, - UnterminatedStringLiteral, + EmptyInterpolation, InvalidFloatLiteral, InvalidHexLiteral, InvalidIntLiteral, + InvalidUTF8Sequence, TailingHereDoc, UnexpectedCharacter, UnexpectedEscapeCode, + UnterminatedExtractor, UnterminatedHereDoc, UnterminatedIdentLiteral, + UnterminatedInterpolation, UnterminatedStringLiteral, }; match self { UnterminatedExtractor(_, _, token) | UnterminatedStringLiteral(_, _, token) | UnterminatedInterpolation(_, _, token) + | EmptyInterpolation(_, _, token) | UnterminatedIdentLiteral(_, _, token) | UnterminatedHereDoc(_, _, token) | TailingHereDoc(_, _, token, _) @@ -298,8 +300,9 @@ impl ErrorKind { pub(crate) fn hint(&self) -> Option { use ErrorKind::{ - BadAccessInEvent, BadAccessInGlobal, BadAccessInLocal, MissingFunction, MissingModule, - NoClauseHit, Oops, TypeConflict, UnrecognizedToken, UnterminatedInterpolation, + BadAccessInEvent, BadAccessInGlobal, BadAccessInLocal, EmptyInterpolation, + MissingFunction, MissingModule, NoClauseHit, Oops, TypeConflict, UnrecognizedToken, + UnterminatedInterpolation, }; match self { UnrecognizedToken(outer, inner, t, _) if t.is_empty() && inner.0.absolute() == outer.1.absolute() => Some("It looks like a `;` is missing at the end of the script".into()), @@ -312,8 +315,8 @@ impl ErrorKind { _ => None } } - UnterminatedInterpolation(_, _, _) => { - Some("Did you mean to write a literal '#{' ? Escape it as '\\#{'.".to_string()) + UnterminatedInterpolation(_, _, _) | EmptyInterpolation(_, _, _) => { + Some("Did you mean to write a literal '#{'? Escape it as '\\#{'.".to_string()) } BadAccessInLocal(_, _, key, _) if key == "nil" => { Some("Did you mean null?".to_owned()) @@ -551,6 +554,10 @@ error_chain! { description("Unterminated String interpolation") display("It looks like you forgot to terminate a string interpolation with a closing '}}'") } + EmptyInterpolation(expr: Range, inner: Range, string_with_interpolation: UnfinishedToken) { + description("Empty interpolation") + display("You have an interpolation without content.") + } UnterminatedIdentLiteral(expr: Range, inner: Range, ident: UnfinishedToken) { diff --git a/tremor-script/src/lexer.rs b/tremor-script/src/lexer.rs index 340f58ed3e..baef90d56b 100644 --- a/tremor-script/src/lexer.rs +++ b/tremor-script/src/lexer.rs @@ -1857,13 +1857,34 @@ impl<'input> Lexer<'input> { *segment_start = end_inner; res.push(self.spanned2(*segment_start, *end, Token::Interpol)); let mut pcount = 0; + let mut first = true; loop { match self.next() { Some(Ok(s)) => { match &s.value { Token::RBrace if pcount == 0 => { + let start = *segment_start; *segment_start = s.span.pp_end; + res.push(s); + if first { + let end_location = *segment_start; + let token_str = self + .slice_full_lines(&total_start, &end_location) + .unwrap_or_else(|| format!("{}{}", error_prefix, content)); + + let unfinished_token = UnfinishedToken::new( + Range::from((start, end_location)), + token_str, + ); + + return Err(ErrorKind::EmptyInterpolation( + Range::from((total_start, end_location)), + Range::from((start, end_location)), + unfinished_token, + ) + .into()); + } break; } Token::RBrace => { @@ -1875,6 +1896,7 @@ impl<'input> Lexer<'input> { _ => {} }; res.push(s); + first = false; } // intercept error and extend the token to match this outer heredoc // with interpolation From 31cb0bccc5ca421cfab826abd7edfaf7f7998ba4 Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Wed, 3 Feb 2021 11:36:20 +0100 Subject: [PATCH 7/9] Improve handling of recursion errors Signed-off-by: Heinz N. Gies --- tests/script_errors/new.sh | 5 +++++ tests/script_runtime_error.rs | 1 + .../function_error_1/error.txt | 2 ++ .../function_error_2/error.txt | 2 ++ .../function_error_3/error.txt | 4 +++- .../function_error_n/error.txt | 4 +++- .../recursion_limit/error.txt | 5 +++++ .../script_runtime_errors/recursion_limit/in.xz | Bin 0 -> 60 bytes .../recursion_limit/script.tremor | 6 ++++++ tremor-script/src/errors.rs | 15 +++++++++++---- tremor-script/src/interpreter/imut_expr.rs | 14 +++++++++----- tremor-script/src/registry.rs | 5 ++++- tremor-script/src/registry/custom_fn.rs | 4 +--- 13 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 tests/script_runtime_errors/recursion_limit/error.txt create mode 100644 tests/script_runtime_errors/recursion_limit/in.xz create mode 100644 tests/script_runtime_errors/recursion_limit/script.tremor diff --git a/tests/script_errors/new.sh b/tests/script_errors/new.sh index 1e019772ae..8fc4c56d78 100755 --- a/tests/script_errors/new.sh +++ b/tests/script_errors/new.sh @@ -19,3 +19,8 @@ git add ${TARGET} sed -e '/^ \/\/INSERT/a\ '"${NAME}," ${BASEDIR}/../script_error.rs > ${BASEDIR}/tmp && mv ${BASEDIR}/tmp ${BASEDIR}/../script_error.rs + +for f in ${TARGET}/* +do + echo "$f" +done \ No newline at end of file diff --git a/tests/script_runtime_error.rs b/tests/script_runtime_error.rs index 60a3beea5d..cccb11e397 100644 --- a/tests/script_runtime_error.rs +++ b/tests/script_runtime_error.rs @@ -164,6 +164,7 @@ test_cases!( subslice_no_arr, subslice_out_of_bounds, undefined_local, + recursion_limit, ); // There errors on thise are not optimal diff --git a/tests/script_runtime_errors/function_error_1/error.txt b/tests/script_runtime_errors/function_error_1/error.txt index 60699907e2..8f479b4bb0 100644 --- a/tests/script_runtime_errors/function_error_1/error.txt +++ b/tests/script_runtime_errors/function_error_1/error.txt @@ -1,3 +1,5 @@ Error: + 2 | + 3 | let a = 7; 4 | record::from_array(a) | ^^^^^^^^^^^^^^^^^^^^^ Bad type passed to function record::from_array/1 \ No newline at end of file diff --git a/tests/script_runtime_errors/function_error_2/error.txt b/tests/script_runtime_errors/function_error_2/error.txt index c4d61570a1..b41d245c9a 100644 --- a/tests/script_runtime_errors/function_error_2/error.txt +++ b/tests/script_runtime_errors/function_error_2/error.txt @@ -1,3 +1,5 @@ Error: + 1 | use std::math; + 2 | let a = 7; 3 | math::max("snot", a) | ^^^^^^^^^^^^^^^^^^^^ Bad type passed to function math::max/2 \ No newline at end of file diff --git a/tests/script_runtime_errors/function_error_3/error.txt b/tests/script_runtime_errors/function_error_3/error.txt index c9dcdd2efe..1483b001b7 100644 --- a/tests/script_runtime_errors/function_error_3/error.txt +++ b/tests/script_runtime_errors/function_error_3/error.txt @@ -1,3 +1,5 @@ Error: + 1 | use std::string; + 2 | let a = 7; 3 | string::format(a, "snot", "badger") - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Runtime error in function string::format/3: expected 1st parameter to format to be a format specifier e.g. to print a number use `string::format("{}", 1)` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Runtime error in function string::format/3: expected 1st parameter to format to be a format specifier e.g. to print a number use `string::format("{}", 1)` \ No newline at end of file diff --git a/tests/script_runtime_errors/function_error_n/error.txt b/tests/script_runtime_errors/function_error_n/error.txt index 9d021d9a72..7d1c347518 100644 --- a/tests/script_runtime_errors/function_error_n/error.txt +++ b/tests/script_runtime_errors/function_error_n/error.txt @@ -1,3 +1,5 @@ Error: + 1 | use std::string; + 2 | let a = 7; 3 | string::format(a, "snot", "badger", "cake") - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Runtime error in function string::format/4: expected 1st parameter to format to be a format specifier e.g. to print a number use `string::format("{}", 1)` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Runtime error in function string::format/4: expected 1st parameter to format to be a format specifier e.g. to print a number use `string::format("{}", 1)` \ No newline at end of file diff --git a/tests/script_runtime_errors/recursion_limit/error.txt b/tests/script_runtime_errors/recursion_limit/error.txt new file mode 100644 index 0000000000..ea775368e0 --- /dev/null +++ b/tests/script_runtime_errors/recursion_limit/error.txt @@ -0,0 +1,5 @@ +Error: + 4 | end; + 5 | + 6 | boom(1) + | ^^^^^^^ Recursion limit Reached \ No newline at end of file diff --git a/tests/script_runtime_errors/recursion_limit/in.xz b/tests/script_runtime_errors/recursion_limit/in.xz new file mode 100644 index 0000000000000000000000000000000000000000..63fc3665fe24a1262bcbe6543867eab9bcc899f9 GIT binary patch literal 60 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=85j+@fb6IKJGJdO1xgthrI_yMEv=K^ O_PLf3D9*qV83h3U8V~6J literal 0 HcmV?d00001 diff --git a/tests/script_runtime_errors/recursion_limit/script.tremor b/tests/script_runtime_errors/recursion_limit/script.tremor new file mode 100644 index 0000000000..2ee8da9e07 --- /dev/null +++ b/tests/script_runtime_errors/recursion_limit/script.tremor @@ -0,0 +1,6 @@ +fn boom(x) with + let x = x + 1; + recur(x) +end; + +boom(1) \ No newline at end of file diff --git a/tremor-script/src/errors.rs b/tremor-script/src/errors.rs index 360dd024f7..f6dab6bd16 100644 --- a/tremor-script/src/errors.rs +++ b/tremor-script/src/errors.rs @@ -183,10 +183,11 @@ impl ErrorKind { MissingModule, ModuleNotFound, Msg, NoClauseHit, NoConstsAllowed, NoLocalsAllowed, NoObjectError, NotConstant, NotFound, Oops, ParseIntError, ParserError, PatchKeyExists, PreprocessorError, QueryNodeDuplicateName, QueryNodeReservedName, - QueryStreamNotDefined, RuntimeError, TailingHereDoc, TypeConflict, UnexpectedCharacter, - UnexpectedEndOfStream, UnexpectedEscapeCode, UnrecognizedToken, UnterminatedExtractor, - UnterminatedHereDoc, UnterminatedIdentLiteral, UnterminatedInterpolation, - UnterminatedStringLiteral, UpdateKeyMissing, Utf8Error, ValueError, + QueryStreamNotDefined, RecursionLimit, RuntimeError, TailingHereDoc, TypeConflict, + UnexpectedCharacter, UnexpectedEndOfStream, UnexpectedEscapeCode, UnrecognizedToken, + UnterminatedExtractor, UnterminatedHereDoc, UnterminatedIdentLiteral, + UnterminatedInterpolation, UnterminatedStringLiteral, UpdateKeyMissing, Utf8Error, + ValueError, }; match self { NoClauseHit(outer) @@ -213,6 +214,7 @@ impl ErrorKind { | InvalidDrop(outer, inner) | InvalidEmit(outer, inner) | InvalidRecur(outer, inner) + | RecursionLimit(outer, inner) | InvalidConst(outer, inner) | InvalidMod(outer, inner) | InvalidFn(outer, inner) @@ -529,6 +531,11 @@ error_chain! { description("Can not recur from this location") display("Can not recur from this location") } + RecursionLimit(expr: Range, inner: Range) { + description("Recursion limit Reached") + display("Recursion limit Reached") + } + /* * Lexer, Preprocessor and Parser */ diff --git a/tremor-script/src/interpreter/imut_expr.rs b/tremor-script/src/interpreter/imut_expr.rs index debaa91e06..678e6137aa 100644 --- a/tremor-script/src/interpreter/imut_expr.rs +++ b/tremor-script/src/interpreter/imut_expr.rs @@ -17,7 +17,6 @@ use super::{ test_predicate_expr, AggrType, Env, ExecOpts, LocalStack, FALSE, TRUE, }; -use crate::prelude::*; use crate::registry::{Registry, TremorAggrFnWrapper, RECUR_REF}; use crate::stry; use crate::{ @@ -35,6 +34,7 @@ use crate::{ }, errors::error_oops_err, }; +use crate::{lexer::Range, prelude::*}; use crate::{Object, Value}; use std::borrow::Cow; use std::mem; @@ -536,7 +536,8 @@ where .map(Cow::Owned) .map_err(|e| { let r: Option<&Registry> = None; - e.into_err(self, self, r, &env.meta) + let outer: Range = self.extent(&env.meta).expand_lines(2); + e.into_err(&outer, self, r, &env.meta) }) } @@ -558,7 +559,8 @@ where .map(Cow::Owned) .map_err(|e| { let r: Option<&Registry> = None; - e.into_err(self, self, r, &env.meta) + let outer: Range = self.extent(&env.meta).expand_lines(2); + e.into_err(&outer, self, r, &env.meta) }) } @@ -588,7 +590,8 @@ where .map(Cow::Owned) .map_err(|e| { let r: Option<&Registry> = None; - e.into_err(self, self, r, &env.meta) + let outer: Range = self.extent(&env.meta).expand_lines(2); + e.into_err(&outer, self, r, &env.meta) }) } @@ -616,7 +619,8 @@ where .map(Cow::Owned) .map_err(|e| { let r: Option<&Registry> = None; - e.into_err(self, self, r, &env.meta) + let outer: Range = self.extent(&env.meta).expand_lines(2); + e.into_err(&outer, self, r, &env.meta) }) } diff --git a/tremor-script/src/registry.rs b/tremor-script/src/registry.rs index 67ad6e331a..f0f0beaede 100644 --- a/tremor-script/src/registry.rs +++ b/tremor-script/src/registry.rs @@ -183,6 +183,8 @@ pub enum FunctionError { /// The function was called with a bad arity mfa: MFA, }, + /// Recursion Limit Reached + RecursionLimit, /// A generic error Error(Box), } @@ -211,7 +213,7 @@ impl FunctionError { meta: &NodeMetas, ) -> crate::errors::Error { use FunctionError::{ - BadArity, BadType, Error, MissingFunction, MissingModule, RuntimeError, + BadArity, BadType, Error, MissingFunction, MissingModule, RecursionLimit, RuntimeError, }; let outer = outer.extent(meta); let inner = inner.extent(meta); @@ -239,6 +241,7 @@ impl FunctionError { ErrorKind::MissingFunction(outer, inner, vec![m], f, suggestion).into() } BadType { mfa } => ErrorKind::BadType(outer, inner, mfa.m, mfa.f, mfa.a).into(), + RecursionLimit => ErrorKind::RecursionLimit(outer, inner).into(), Error(e) => *e, } } diff --git a/tremor-script/src/registry/custom_fn.rs b/tremor-script/src/registry/custom_fn.rs index b4e9d169ea..d0fa67c782 100644 --- a/tremor-script/src/registry/custom_fn.rs +++ b/tremor-script/src/registry/custom_fn.rs @@ -194,9 +194,7 @@ impl<'script> CustomFn<'script> { recursion_depth += 1; if recursion_depth == env.recursion_limit { mem::swap(get_args_mut(consts)?, &mut args_const); - return Err(FunctionError::Error(Box::new( - "recursion limit reached".into(), - ))); + return Err(FunctionError::RecursionLimit); } // clear the local variables (that are not the // arguments) From 7a59a06c147ab171988c76a2e47e79bfbc38a97f Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Wed, 3 Feb 2021 11:41:54 +0100 Subject: [PATCH 8/9] Exclude accessors Signed-off-by: Heinz N. Gies --- tremor-script/src/ast/base_expr.rs | 23 +++++++++++++++++++++++ tremor-script/src/errors.rs | 1 + 2 files changed, 24 insertions(+) diff --git a/tremor-script/src/ast/base_expr.rs b/tremor-script/src/ast/base_expr.rs index 9356c519d5..8dd229a203 100644 --- a/tremor-script/src/ast/base_expr.rs +++ b/tremor-script/src/ast/base_expr.rs @@ -114,6 +114,8 @@ impl BaseExpr for (Location, Location) { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for ImutExprInt<'script> { fn s(&self, meta: &NodeMetas) -> Location { match self { @@ -140,6 +142,7 @@ impl<'script> BaseExpr for ImutExprInt<'script> { ImutExprInt::String(e) => e.s(meta), } } + fn e(&self, meta: &NodeMetas) -> Location { match self { ImutExprInt::Binary(e) => e.e(meta), @@ -190,6 +193,8 @@ impl<'script> BaseExpr for ImutExprInt<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for Expr<'script> { fn mid(&self) -> usize { match self { @@ -206,6 +211,8 @@ impl<'script> BaseExpr for Expr<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for PathRaw<'script> { fn s(&self, meta: &NodeMetas) -> Location { match self { @@ -230,6 +237,8 @@ impl<'script> BaseExpr for PathRaw<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for Path<'script> { fn mid(&self) -> usize { match self { @@ -241,6 +250,8 @@ impl<'script> BaseExpr for Path<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for Segment<'script> { fn mid(&self) -> usize { match self { @@ -252,6 +263,8 @@ impl<'script> BaseExpr for Segment<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for ImutExprRaw<'script> { fn mid(&self) -> usize { 0 @@ -296,12 +309,16 @@ impl<'script> BaseExpr for ImutExprRaw<'script> { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl BaseExpr for TestExpr { fn mid(&self) -> usize { self.mid } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl BaseExpr for TestExprRaw { fn s(&self, _meta: &NodeMetas) -> Location { self.start @@ -315,18 +332,24 @@ impl BaseExpr for TestExprRaw { } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl BaseExpr for InvokeAggr { fn mid(&self) -> usize { self.mid } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for GroupBy<'script> { fn mid(&self) -> usize { self.0.mid() } } +// This is a simple accessor +#[cfg(not(tarpaulin_include))] impl<'script> BaseExpr for GroupByInt<'script> { fn mid(&self) -> usize { match self { diff --git a/tremor-script/src/errors.rs b/tremor-script/src/errors.rs index f6dab6bd16..51ce8c0dea 100644 --- a/tremor-script/src/errors.rs +++ b/tremor-script/src/errors.rs @@ -169,6 +169,7 @@ impl ErrorKind { pub(crate) fn cu(&self) -> usize { self.expr().0.map(Range::cu).unwrap_or_default() } + #[allow(clippy::too_many_lines)] pub(crate) fn expr(&self) -> ErrorLocation { use ErrorKind::{ AccessError, AggrInAggr, ArrayOutOfRange, AssignIntoArray, AssignToConst, From 998226396aaef5b8842ba441b125464636aeb4ca Mon Sep 17 00:00:00 2001 From: "Heinz N. Gies" Date: Wed, 3 Feb 2021 13:25:08 +0100 Subject: [PATCH 9/9] Add missing test Signed-off-by: Heinz N. Gies --- tests/script_error.rs | 1 + tests/script_errors/string_interpolation_empty/error.txt | 4 ++++ tests/script_errors/string_interpolation_empty/script.tremor | 1 + 3 files changed, 6 insertions(+) create mode 100644 tests/script_errors/string_interpolation_empty/error.txt create mode 100644 tests/script_errors/string_interpolation_empty/script.tremor diff --git a/tests/script_error.rs b/tests/script_error.rs index dda2ded14b..919d9ee3f0 100644 --- a/tests/script_error.rs +++ b/tests/script_error.rs @@ -134,6 +134,7 @@ test_cases!( pp_cyclic, pp_nest_cyclic, //INSERT + string_interpolation_empty, fn_bad_recur, script_without_newline, lexer_invalid_hex2, diff --git a/tests/script_errors/string_interpolation_empty/error.txt b/tests/script_errors/string_interpolation_empty/error.txt new file mode 100644 index 0000000000..3a1ec64d31 --- /dev/null +++ b/tests/script_errors/string_interpolation_empty/error.txt @@ -0,0 +1,4 @@ +Error: + 1 | "#{}" + | ^^^ You have an interpolation without content. + | NOTE: Did you mean to write a literal '#{'? Escape it as '\#{'. \ No newline at end of file diff --git a/tests/script_errors/string_interpolation_empty/script.tremor b/tests/script_errors/string_interpolation_empty/script.tremor new file mode 100644 index 0000000000..7fc368d114 --- /dev/null +++ b/tests/script_errors/string_interpolation_empty/script.tremor @@ -0,0 +1 @@ +"#{}" \ No newline at end of file