parser recovery at item level

sway-ast/src/item/mod.rs

@@ -1,3 +1,5 @@
+use sway_error::handler::ErrorEmitted;
+
 use crate::priv_prelude::*;
 
 pub mod item_abi;
@@ -38,6 +40,8 @@ pub enum ItemKind {
     Storage(ItemStorage),
     Configurable(ItemConfigurable),
     TypeAlias(ItemTypeAlias),
+    // to handle parser recovery: Error represents an incomplete item
+    Error(Box<[Span]>, #[serde(skip_serializing)] ErrorEmitted),
 }
 
 impl Spanned for ItemKind {
@@ -55,6 +59,7 @@ impl Spanned for ItemKind {
             ItemKind::Storage(item_storage) => item_storage.span(),
             ItemKind::Configurable(item_configurable) => item_configurable.span(),
             ItemKind::TypeAlias(item_type_alias) => item_type_alias.span(),
+            ItemKind::Error(spans, _) => Span::join_all(spans.iter().cloned()),
         }
     }
 }

sway-core/src/transform/to_parsed_lang/convert_parse_tree.rs

@@ -222,6 +222,9 @@ fn item_to_ast_nodes(
                 attributes,
             )?,
         )),
+        ItemKind::Error(spans, error) => {
+            vec![AstNodeContent::Error(spans, error)]
+        }
     };
 
     Ok(contents

sway-error/src/parser_error.rs

@@ -24,6 +24,8 @@ pub enum ParseErrorKind {
     InvalidLiteralFieldName,
     #[error("Invalid statement.")]
     InvalidStatement,
+    #[error("Invalid item.")]
+    InvalidItem,
     #[error("Integer field names cannot have type suffixes.")]
     IntFieldWithTypeSuffix,
     #[error("Expected a field name.")]

sway-lsp/src/traverse/lexed_tree.rs

@@ -89,6 +89,7 @@ impl Parse for ItemKind {
             ItemKind::TypeAlias(item_type_alias) => {
                 item_type_alias.parse(ctx);
             }
+            ItemKind::Error(_, _) => {}
         }
     }
 }

sway-parse/src/attribute.rs

@@ -55,6 +55,7 @@ impl<T: Parse> Parse for Annotated<T> {
                 ),
             });
         }
+
         while let Some(attr) = parser.guarded_parse::<HashToken, _>()? {
             attribute_list.push(attr);
         }
@@ -64,14 +65,38 @@ impl<T: Parse> Parse for Annotated<T> {
             Err(error)
         } else {
             // Parse the `T` value.
-            let value = parser.parse()?;
+            let value = match parser.parse_with_recovery() {
+                Ok(value) => value,
+                Err(r) => {
+                    let (spans, error) =
+                        r.recover_at_next_line_with_fallback_error(ParseErrorKind::InvalidItem);
+                    if let Some(error) = T::error(spans, error) {
+                        error
+                    } else {
+                        Err(error)?
+                    }
+                }
+            };
 
             Ok(Annotated {
                 attribute_list,
                 value,
             })
         }
     }
+
+    fn error(
+        spans: Box<[sway_types::Span]>,
+        error: sway_error::handler::ErrorEmitted,
+    ) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        T::error(spans, error).map(|value| Annotated {
+            attribute_list: vec![],
+            value,
+        })
+    }
 }
 
 impl Parse for AttributeDecl {

sway-parse/src/expr/mod.rs

@@ -132,20 +132,29 @@ impl ParseToEnd for CodeBlockContents {
         mut parser: Parser<'a, '_>,
     ) -> ParseResult<(CodeBlockContents, ParserConsumed<'a>)> {
         let mut statements = Vec::new();
+
         let (final_expr_opt, consumed) = loop {
             if let Some(consumed) = parser.check_empty() {
                 break (None, consumed);
             }
-            match parse_stmt(&mut parser)? {
-                StmtOrTail::Stmt(s) => statements.push(s),
-                StmtOrTail::Tail(e, c) => break (Some(e), c),
+
+            match parser.parse_fn_with_recovery(parse_stmt) {
+                Ok(StmtOrTail::Stmt(s)) => statements.push(s),
+                Ok(StmtOrTail::Tail(e, c)) => break (Some(e), c),
+                Err(r) => {
+                    let (spans, error) = r
+                        .recover_at_next_line_with_fallback_error(ParseErrorKind::InvalidStatement);
+                    statements.push(Statement::Error(spans, error));
+                }
             }
         };
+
         let code_block_contents = CodeBlockContents {
             statements,
             final_expr_opt,
             span: parser.full_span().clone(),
         };
+
         Ok((code_block_contents, consumed))
     }
 }
@@ -187,14 +196,8 @@ fn parse_stmt<'a>(parser: &mut Parser<'a, '_>) -> ParseResult<StmtOrTail<'a>> {
     }
 
     // Try a `let` statement.
-    match parser.guarded_parse_with_recovery::<LetToken, StatementLet>() {
-        Ok(None) => {}
-        Ok(Some(item)) => return stmt(Statement::Let(item)),
-        Err(r) => {
-            let (spans, error) =
-                r.recover_at_next_line_with_fallback_error(ParseErrorKind::InvalidStatement);
-            return stmt(Statement::Error(spans, error));
-        }
+    if let Some(item) = parser.guarded_parse::<LetToken, StatementLet>()? {
+        return stmt(Statement::Let(item));
     }
 
     // Try an `expr;` statement.

sway-parse/src/item/mod.rs

@@ -76,6 +76,16 @@ impl Parse for ItemKind {
 
         Ok(kind)
     }
+
+    fn error(
+        spans: Box<[sway_types::Span]>,
+        error: sway_error::handler::ErrorEmitted,
+    ) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        Some(ItemKind::Error(spans, error))
+    }
 }
 
 impl Parse for TypeField {

sway-parse/src/parse.rs

@@ -9,6 +9,16 @@ pub trait Parse {
     fn parse(parser: &mut Parser) -> ParseResult<Self>
     where
         Self: Sized;
+
+    fn error(
+        #[allow(clippy::boxed_local)] _spans: Box<[sway_types::Span]>,
+        _error: sway_error::handler::ErrorEmitted,
+    ) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        None
+    }
 }
 
 pub trait Peek {
@@ -95,8 +105,19 @@ where
             if let Some(consumed) = parser.check_empty() {
                 return Ok((ret, consumed));
             }
-            let value = parser.parse()?;
-            ret.push(value);
+
+            match parser.parse_with_recovery() {
+                Ok(value) => ret.push(value),
+                Err(r) => {
+                    let (spans, error) =
+                        r.recover_at_next_line_with_fallback_error(ParseErrorKind::InvalidItem);
+                    if let Some(error) = T::error(spans, error) {
+                        ret.push(error);
+                    } else {
+                        Err(error)?
+                    }
+                }
+            }
         }
     }
 }

sway-parse/src/parser.rs

@@ -76,6 +76,77 @@ impl<'a, 'e> Parser<'a, 'e> {
         Peeker::with(self.token_trees).map(|(v, _)| v)
     }
 
+    pub fn parse_fn_with_recovery<
+        'original,
+        T,
+        F: FnOnce(&mut Parser<'a, '_>) -> ParseResult<T>,
+    >(
+        &'original mut self,
+        f: F,
+    ) -> Result<T, Recoverer<'original, 'a, 'e>> {
+        let handler = Handler::default();
+        let mut fork = Parser {
+            token_trees: self.token_trees,
+            full_span: self.full_span.clone(),
+            handler: &handler,
+        };
+
+        match f(&mut fork) {
+            Ok(result) => {
+                self.token_trees = fork.token_trees;
+                self.handler.append(handler);
+                Ok(result)
+            }
+            Err(error) => {
+                let Parser {
+                    token_trees,
+                    full_span,
+                    ..
+                } = fork;
+                Err(Recoverer {
+                    original: RefCell::new(self),
+                    handler,
+                    fork_token_trees: token_trees,
+                    fork_full_span: full_span,
+                    error,
+                })
+            }
+        }
+    }
+
+    pub fn parse_with_recovery<'original, T: Parse>(
+        &'original mut self,
+    ) -> Result<T, Recoverer<'original, 'a, 'e>> {
+        let handler = Handler::default();
+        let mut fork = Parser {
+            token_trees: self.token_trees,
+            full_span: self.full_span.clone(),
+            handler: &handler,
+        };
+
+        match fork.parse() {
+            Ok(result) => {
+                self.token_trees = fork.token_trees;
+                self.handler.append(handler);
+                Ok(result)
+            }
+            Err(error) => {
+                let Parser {
+                    token_trees,
+                    full_span,
+                    ..
+                } = fork;
+                Err(Recoverer {
+                    original: RefCell::new(self),
+                    handler,
+                    fork_token_trees: token_trees,
+                    fork_full_span: full_span,
+                    error,
+                })
+            }
+        }
+    }
+
     /// This function do three things
     /// 1 - it peeks P;
     /// 2 - it forks the current parse, and try to parse
@@ -376,10 +447,19 @@ impl<'original, 'a, 'e> Recoverer<'original, 'a, 'e> {
         &self,
         kind: ParseErrorKind,
     ) -> (Box<[Span]>, ErrorEmitted) {
-        let last_token_span = self.last_consumed_token().span();
-        let last_token_span_line = last_token_span.start_pos().line_col().0;
+        let line = if self.fork_token_trees.is_empty() {
+            None
+        } else {
+            self.last_consumed_token()
+                .map(|x| x.span())
+                .or_else(|| self.fork_token_trees.get(0).map(|x| x.span()))
+                .map(|x| x.start_pos().line_col().0)
+        };
+
         self.recover(|p| {
-            p.consume_while_line_equals(last_token_span_line);
+            if let Some(line) = line {
+                p.consume_while_line_equals(line);
+            }
             if !p.has_errors() {
                 p.emit_error_with_span(kind, self.diff_span(p));
             }
@@ -413,15 +493,18 @@ impl<'original, 'a, 'e> Recoverer<'original, 'a, 'e> {
 
     /// This is the last consumed token of the forked parser. This the token
     /// immediately before the forked parser head.
-    pub fn last_consumed_token(&self) -> &GenericTokenTree<TokenStream> {
+    pub fn last_consumed_token(&self) -> Option<&GenericTokenTree<TokenStream>> {
+        let fork_head_span = self.fork_token_trees.get(0)?.span();
+
         // find the last token consumed by the fork
         let original = self.original.borrow();
         let fork_pos = original
             .token_trees
             .iter()
-            .position(|x| x.span() == self.fork_token_trees[0].span())
-            .unwrap();
-        &original.token_trees[fork_pos - 1]
+            .position(|x| x.span() == fork_head_span)?;
+
+        let before_fork_pos = fork_pos.checked_sub(1)?;
+        original.token_trees.get(before_fork_pos)
     }
 
     /// This return a span encopassing all tokens that were consumed by the `p` since the start

swayfmt/src/module/item.rs

@@ -23,6 +23,7 @@ impl Format for ItemKind {
             Storage(item_storage) => item_storage.format(formatted_code, formatter),
             Configurable(item_configurable) => item_configurable.format(formatted_code, formatter),
             TypeAlias(item_type_alias) => item_type_alias.format(formatted_code, formatter),
+            Error(_, _) => Ok(()),
         }
     }
 }
@@ -42,6 +43,9 @@ impl LeafSpans for ItemKind {
             Use(item_use) => item_use.leaf_spans(),
             Configurable(item_configurable) => item_configurable.leaf_spans(),
             TypeAlias(item_type_alias) => item_type_alias.leaf_spans(),
+            Error(spans, _) => {
+                vec![sway_types::Span::join_all(spans.iter().cloned()).into()]
+            }
         }
     }
 }

test/src/e2e_vm_tests/test_programs/should_fail/recover_invalid_items/Forc.lock

@@ -0,0 +1,3 @@
+[[package]]
+name = 'recover_invalid_items'
+source = 'member'

test/src/e2e_vm_tests/test_programs/should_fail/recover_invalid_items/Forc.toml

@@ -0,0 +1,6 @@
+[project]
+name = "recover_invalid_items"
+authors = ["Fuel Labs <contact@fuel.sh>"]
+entry = "main.sw"
+license = "Apache-2.0"
+implicit-std = false

test/src/e2e_vm_tests/test_programs/should_fail/recover_invalid_items/src/main.sw

@@ -0,0 +1,16 @@
+script;
+
+random_words
+
+f9
+
+
+fn good1() {
+
+}
+
+fn a
+
+fn main() {
+    good1();
+}

test/src/e2e_vm_tests/test_programs/should_fail/recover_invalid_items/test.toml

@@ -0,0 +1,5 @@
+category = "fail"
+
+# check: $()Expected an item
+# check: $()Expected an item
+# check: $()Expected an opening parenthesis

test/src/e2e_vm_tests/test_programs/should_fail/recover_unclosed_multiline_comment/src/main.sw

@@ -1,9 +1,17 @@
 contract;
 
+/
+/ Comments
+
 mod foo;
 mod bar;
 mod baz;
 
+fn f() {
+    /
+    f2();
+}
+
 fn a() -> bool { 0 } // recovery witness
 
 /*