Fix number literal errors, Allow literals of i64::MIN, fix scientific…

… float literals.

Signed-off-by: Matthias Wahl <mwahl@wayfair.com>

.vscode/launch.json

@@ -20,19 +20,19 @@
         }
       },
       "args": [
-        "server",
         //server run -f temp/sample.yaml temp/sample.trickle -l temp/logger.yaml
+        //"server",
         "run",
-        "-f",
-        "temp/sample.yaml",
-        "temp/sample.trickle",
-        "-l",
-        "temp/logger.yaml",
+        //"-f",
+        //"temp/sample.yaml",
+        //"temp/sample.trickle",
+        //"-l",
+        //"temp/logger.yaml",
         //"tremor-cli/tests/cli/data/script_with_error.tremor",
         //"-i",
         //"tremor-cli/tests/cli/data/input.json",
-        //"temp/sample.trickle",
-        //"tests/scripts/heredoc_interpolation_quotes/script.tremor"
+        "temp/end.tremor",
+        //"tests/script_errors/lexer_invalid_float/script.tremor"
         //"-f",
         //"../tremor-www-docs/docs/workshop/examples/35_reverse_proxy_load_balancing/etc/tremor/config/config.yaml",
         // "../tremor-www-docs/docs/workshop/examples/35_reverse_proxy_load_balancing/etc/tremor/config/request_handling.trickle",

src/offramp.rs

@@ -212,7 +212,7 @@ impl Manager {
             )
             .await
         {
-            error!("Failed to create onramp {}: {}", id, e);
+            error!("Failed to create offramp {}: {}", id, e);
             return Err(e);
         }
         // merge channels and prioritize contraflow/insight events

tests/script_error.rs

@@ -134,6 +134,11 @@ test_cases!(
     pp_cyclic,
     pp_nest_cyclic,
     //INSERT
+lexer_invalid_hex2,
+    lexer_invalid_int_invalid_char,
+    lexer_invalid_hex,
+    lexer_invalid_float,
+    lexer_invalid_int,
     string_interpolation_eof,
     string_interpolation_invalid_utf8,
     string_interpolation_escape,

tests/script_errors/lexer_invalid_float/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | 12.0_5
+      |     ^ An unexpected character '_' was found

tests/script_errors/lexer_invalid_float/script.tremor

@@ -0,0 +1 @@
+12.0_5

tests/script_errors/lexer_invalid_hex/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | let invalid = 0x;
+      |               ^^^ An invalid hexadecimal

tests/script_errors/lexer_invalid_hex/script.tremor

@@ -0,0 +1 @@
+let invalid = 0x;

tests/script_errors/lexer_invalid_hex2/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | -0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+      | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An invalid hexadecimal

tests/script_errors/lexer_invalid_hex2/script.tremor

@@ -0,0 +1 @@
+-0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF

tests/script_errors/lexer_invalid_int/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | -161390617380431786853494948250188242145606612051826469551916209783790476376052574664352834580008614464743948248296718335
+      | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An invalid integer literal

tests/script_errors/lexer_invalid_int/script.tremor

@@ -0,0 +1 @@
+-161390617380431786853494948250188242145606612051826469551916209783790476376052574664352834580008614464743948248296718335

tests/script_errors/lexer_invalid_int_invalid_char/error.txt

@@ -0,0 +1,3 @@
+Error: 
+    1 | 12345_
+      |      ^ An unexpected character '_' was found

tests/script_errors/lexer_invalid_int_invalid_char/script.tremor

@@ -0,0 +1 @@
+12345_

tremor-pipeline/src/executable_graph.rs

@@ -817,7 +817,7 @@ mod test {
             last_metrics: 0,
             metric_interval: Some(1),
             insights: vec![],
-            source: None, // TODO
+            source: None,
             dot: String::from(""),
         };
 

tremor-script/Cargo.toml

@@ -63,6 +63,7 @@ criterion = "0.3"
 float-cmp = "0.8"
 matches = "0.1"
 pretty_assertions = "0.6.1"
+proptest = "0.10"
 
 [features]
 fns = []

tremor-script/src/errors.rs

@@ -215,9 +215,9 @@ impl ErrorKind {
             | DoubleConst(outer, inner, _)
             | DoubleStream(outer, inner, _)
             | InvalidExtractor(outer, inner, _, _, _)
-            | InvalidFloatLiteral(outer, inner)
-            | InvalidHexLiteral(outer, inner)
-            | InvalidIntLiteral(outer, inner)
+            | InvalidFloatLiteral(outer, inner, _)
+            | InvalidHexLiteral(outer, inner, _)
+            | InvalidIntLiteral(outer, inner, _)
             | InvalidToken(outer, inner)
             | InvalidUnary(outer, inner, _, _)
             | MergeTypeConflict(outer, inner, _, _)
@@ -229,7 +229,7 @@ impl ErrorKind {
             | NoConstsAllowed(outer, inner)
             | RuntimeError(outer, inner, _, _, _, _)
             | TypeConflict(outer, inner, _, _)
-            | UnexpectedCharacter(outer, inner, _)
+            | UnexpectedCharacter(outer, inner, _, _)
             | UnexpectedEscapeCode(outer, inner, _, _)
             | InvalidUTF8Sequence(outer, inner, _)
             | UnrecognizedToken(outer, inner, _, _)
@@ -266,7 +266,8 @@ impl ErrorKind {
     }
     pub(crate) fn token(&self) -> Option<UnfinishedToken> {
         use ErrorKind::{
-            InvalidUTF8Sequence, TailingHereDoc, UnexpectedEscapeCode, UnterminatedExtractor,
+            InvalidFloatLiteral, InvalidHexLiteral, InvalidIntLiteral, InvalidUTF8Sequence,
+            TailingHereDoc, UnexpectedCharacter, UnexpectedEscapeCode, UnterminatedExtractor,
             UnterminatedHereDoc, UnterminatedIdentLiteral, UnterminatedInterpolation,
             UnterminatedStringLiteral,
         };
@@ -278,6 +279,10 @@ impl ErrorKind {
             | UnterminatedHereDoc(_, _, token)
             | TailingHereDoc(_, _, token, _)
             | InvalidUTF8Sequence(_, _, token)
+            | UnexpectedCharacter(_, _, token, _)
+            | InvalidHexLiteral(_, _, token)
+            | InvalidIntLiteral(_, _, token)
+            | InvalidFloatLiteral(_, _, token)
             | UnexpectedEscapeCode(_, _, token, _) => Some(token.clone()),
             _ => None,
         }
@@ -543,7 +548,7 @@ error_chain! {
                 display("It looks like you forgot to terminate an ident with a closing '`'")
         }
 
-        UnexpectedCharacter(expr: Range, inner: Range, found: char){
+        UnexpectedCharacter(expr: Range, inner: Range, token: UnfinishedToken, found: char){
             description("An unexpected character was found")
                 display("An unexpected character '{}' was found", found)
         }
@@ -558,18 +563,18 @@ error_chain! {
                 display("An invalid UTF8 escape sequence was found")
 
         }
-        InvalidHexLiteral(expr: Range, inner: Range){
+        InvalidHexLiteral(expr: Range, inner: Range, token: UnfinishedToken){
             description("An invalid hexadecimal")
                 display("An invalid hexadecimal")
 
         }
 
-        InvalidIntLiteral(expr: Range, inner: Range) {
+        InvalidIntLiteral(expr: Range, inner: Range, token: UnfinishedToken) {
             description("An invalid integer literal")
                 display("An invalid integer literal")
 
         }
-        InvalidFloatLiteral(expr: Range, inner: Range) {
+        InvalidFloatLiteral(expr: Range, inner: Range, token: UnfinishedToken) {
             description("An invalid float literal")
                 display("An invalid float literal")