Merge pull request #3578 from onflow/supun/improve-parser-errors

Always return parser errors gracefully

runtime/environment.go

@@ -522,9 +522,15 @@ func (e *interpreterEnvironment) parseAndCheckProgram(
  err error,
 ) {
  wrapParsingCheckingError := func(err error) error {
- return &ParsingCheckingError{
- Err: err,
- Location: location,
+ switch err.(type) {
+ // Wrap only parsing and checking errors.
+ case *sema.CheckerError, parser.Error:
+ return &ParsingCheckingError{
+ Err: err,
+ Location: location,
+ }
+ default:
+ return err
  }
  }
 

runtime/error_test.go

@@ -59,6 +59,7 @@ func TestRuntimeError(t *testing.T) {
  Location: location,
  },
  )
+
  require.EqualError(
  t,
  err,

runtime/old_parser/parser.go

@@ -91,8 +91,13 @@ func Parse[T any](
  config Config,
 ) (result T, errors []error) {
  // create a lexer, which turns the input string into tokens
- tokens := lexer.Lex(input, memoryGauge)
+ tokens, err := lexer.Lex(input, memoryGauge)
+ if err != nil {
+ errors = append(errors, err)
+ return
+ }
  defer tokens.Reclaim()
+
  return ParseTokenStream(
  memoryGauge,
  tokens,
@@ -637,8 +642,12 @@ func ParseArgumentList(
 }
 
 func ParseProgram(memoryGauge common.MemoryGauge, code []byte, config Config) (program *ast.Program, err error) {
- tokens := lexer.Lex(code, memoryGauge)
+ tokens, err := lexer.Lex(code, memoryGauge)
+ if err != nil {
+ return
+ }
  defer tokens.Reclaim()
+
  return ParseProgramFromTokenStream(memoryGauge, tokens, config)
 }
 

runtime/old_parser/parser_test.go

@@ -732,18 +732,11 @@ func TestParseArgumentList(t *testing.T) {
  gauge := makeLimitingMemoryGauge()
  gauge.Limit(common.MemoryKindTypeToken, 0)
 
- var panicMsg any
- (func() {
- defer func() {
- panicMsg = recover()
- }()
+ _, err := ParseArgumentList(gauge, []byte(`(1, b: true)`), Config{})
+ require.Len(t, err, 1)
+ require.IsType(t, errors.MemoryError{}, err[0])
 
- ParseArgumentList(gauge, []byte(`(1, b: true)`), Config{})
- })()
-
- require.IsType(t, errors.MemoryError{}, panicMsg)
-
- fatalError, _ := panicMsg.(errors.MemoryError)
+ fatalError, _ := err[0].(errors.MemoryError)
  var expectedError limitingMemoryGaugeError
  assert.ErrorAs(t, fatalError, &expectedError)
  })

runtime/parser/expression_test.go

@@ -324,17 +324,11 @@ func TestParseAdvancedExpression(t *testing.T) {
  gauge.debug = true
  gauge.Limit(common.MemoryKindPosition, 11)
 
- var panicMsg any
- (func() {
- defer func() {
- panicMsg = recover()
- }()
- ParseExpression(gauge, []byte("1 < 2"), Config{})
- })()
+ _, errs := ParseExpression(gauge, []byte("1 < 2"), Config{})
+ require.Len(t, errs, 1)
+ require.IsType(t, errors.MemoryError{}, errs[0])
 
- require.IsType(t, errors.MemoryError{}, panicMsg)
-
- fatalError, _ := panicMsg.(errors.MemoryError)
+ fatalError, _ := errs[0].(errors.MemoryError)
  var expectedError limitingMemoryGaugeError
  assert.ErrorAs(t, fatalError, &expectedError)
  })
@@ -346,18 +340,11 @@ func TestParseAdvancedExpression(t *testing.T) {
  gauge := makeLimitingMemoryGauge()
  gauge.Limit(common.MemoryKindIntegerExpression, 1)
 
- var panicMsg any
- (func() {
- defer func() {
- panicMsg = recover()
- }()
-
- ParseExpression(gauge, []byte("1 < 2 > 3"), Config{})
- })()
-
- require.IsType(t, errors.MemoryError{}, panicMsg)
+ _, errs := ParseExpression(gauge, []byte("1 < 2 > 3"), Config{})
+ require.Len(t, errs, 1)
+ require.IsType(t, errors.MemoryError{}, errs[0])
 
- fatalError, _ := panicMsg.(errors.MemoryError)
+ fatalError, _ := errs[0].(errors.MemoryError)
  var expectedError limitingMemoryGaugeError
  assert.ErrorAs(t, fatalError, &expectedError)
  })

runtime/parser/lexer/lexer.go

@@ -144,13 +144,13 @@ var pool = sync.Pool{
  },
 }
 
-func Lex(input []byte, memoryGauge common.MemoryGauge) TokenStream {
+func Lex(input []byte, memoryGauge common.MemoryGauge) (TokenStream, error) {
  l := pool.Get().(*lexer)
  l.clear()
  l.memoryGauge = memoryGauge
  l.input = input
- l.run(rootState)
- return l
+ err := l.run(rootState)
+ return l, err
 }
 
 // run executes the stateFn, which will scan the runes in the input
@@ -162,32 +162,29 @@ func Lex(input []byte, memoryGauge common.MemoryGauge) TokenStream {
 // stateFn is returned, which for example happens when reaching the end of the file.
 //
 // When all stateFn have been executed, an EOF token is emitted.
-func (l *lexer) run(state stateFn) {
+func (l *lexer) run(state stateFn) (err error) {
 
  // catch panic exceptions, emit it to the tokens channel before
  // closing it
  defer func() {
  if r := recover(); r != nil {
- var err error
  switch r := r.(type) {
- case errors.MemoryError, errors.InternalError:
- // fatal errors and internal errors percolates up.
- // Note: not all fatal errors are internal errors.
- // e.g: memory limit exceeding is a fatal error, but also a user error.
- panic(r)
+ // fatal errors and internal errors percolates up.
+ // Note: not all fatal errors are internal errors.
+ // e.g: memory limit exceeding is a fatal error, but also a user error.
  case error:
  err = r
  default:
  err = fmt.Errorf("lexer: %v", r)
  }
-
- l.emitError(err)
  }
  }()
 
  for state != nil {
  state = state(l)
  }
+
+ return
 }
 
 // next decodes the next rune (UTF8 character) from the input string.

runtime/parser/lexer/lexer_test.go

@@ -63,7 +63,10 @@ func testLex(t *testing.T, input string, expected []token) {
 
  bytes := []byte(input)
 
- withTokens(Lex(bytes, nil), func(actualTokens []Token) {
+ tokenStream, err := Lex(bytes, nil)
+ require.NoError(t, err)
+
+ withTokens(tokenStream, func(actualTokens []Token) {
  utils.AssertEqualWithDiff(t, expectedTokens, actualTokens)
 
  require.Len(t, actualTokens, len(expectedTokens))
@@ -2385,7 +2388,8 @@ func TestRevert(t *testing.T) {
 
  t.Parallel()
 
- tokenStream := Lex([]byte("1 2 3"), nil)
+ tokenStream, err := Lex([]byte("1 2 3"), nil)
+ require.NoError(t, err)
 
  // Assert all tokens
 
@@ -2550,7 +2554,8 @@ func TestEOFsAfterError(t *testing.T) {
 
  t.Parallel()
 
- tokenStream := Lex([]byte(`1 ''`), nil)
+ tokenStream, err := Lex([]byte(`1 ''`), nil)
+ require.NoError(t, err)
 
  // Assert all tokens
 
@@ -2613,7 +2618,8 @@ func TestEOFsAfterEmptyInput(t *testing.T) {
 
  t.Parallel()
 
- tokenStream := Lex(nil, nil)
+ tokenStream, err := Lex(nil, nil)
+ require.NoError(t, err)
 
  // Assert EOFs keep on being returned for Next()
  // at the end of the stream
@@ -2644,10 +2650,6 @@ func TestLimit(t *testing.T) {
 
  code := b.String()
 
- assert.PanicsWithValue(t,
- TokenLimitReachedError{},
- func() {
- _ = Lex([]byte(code), nil)
- },
- )
+ _, err := Lex([]byte(code), nil)
+ require.ErrorAs(t, err, &TokenLimitReachedError{})
 }

runtime/parser/parser.go

@@ -100,8 +100,13 @@ func Parse[T any](
  config Config,
 ) (result T, errors []error) {
  // create a lexer, which turns the input string into tokens
- tokens := lexer.Lex(input, memoryGauge)
+ tokens, err := lexer.Lex(input, memoryGauge)
+ if err != nil {
+ errors = append(errors, err)
+ return
+ }
  defer tokens.Reclaim()
+
  return ParseTokenStream(
  memoryGauge,
  tokens,
@@ -127,29 +132,34 @@ func ParseTokenStream[T any](
 
  defer func() {
  if r := recover(); r != nil {
+ var err error
  switch r := r.(type) {
  case ParseError:
  // Report parser errors.
  p.report(r)
 
  // Do not treat non-parser errors as syntax errors.
- case errors.InternalError, errors.UserError:
- // Also do not wrap non-parser errors, that are already
- // known cadence errors. i.e: internal errors / user errors.
- // e.g: `errors.MemoryError`
- panic(r)
+ // Also do not wrap non-parser errors, that are already
+ // known cadence errors. i.e: internal errors / user errors.
+ // e.g: `errors.MemoryError`
+ case errors.UserError:
+ err = r
+ case errors.InternalError:
+ err = r
  case error:
  // Any other error/panic is an internal error.
  // Thus, wrap with an UnexpectedError to mark it as an internal error
  // and propagate up the call stack.
- panic(errors.NewUnexpectedErrorFromCause(r))
+ err = errors.NewUnexpectedErrorFromCause(r)
  default:
- panic(errors.NewUnexpectedError("parser: %v", r))
+ err = errors.NewUnexpectedError("parser: %v", r)
  }
 
  var zero T
  result = zero
  errs = p.errors
+
+ errs = append(errs, err)
  }
 
  for _, bufferedErrors := range p.bufferedErrorsStack {
@@ -677,8 +687,12 @@ func ParseArgumentList(
 }
 
 func ParseProgram(memoryGauge common.MemoryGauge, code []byte, config Config) (program *ast.Program, err error) {
- tokens := lexer.Lex(code, memoryGauge)
+ tokens, err := lexer.Lex(code, memoryGauge)
+ if err != nil {
+ return
+ }
  defer tokens.Reclaim()
+
  return ParseProgramFromTokenStream(memoryGauge, tokens, config)
 }
 

runtime/parser/parser_test.go

@@ -734,18 +734,12 @@ func TestParseArgumentList(t *testing.T) {
  gauge := makeLimitingMemoryGauge()
  gauge.Limit(common.MemoryKindTypeToken, 0)
 
- var panicMsg any
- (func() {
- defer func() {
- panicMsg = recover()
- }()
+ _, errs := ParseArgumentList(gauge, []byte(`(1, b: true)`), Config{})
+ require.Len(t, errs, 1)
 
- ParseArgumentList(gauge, []byte(`(1, b: true)`), Config{})
- })()
+ require.IsType(t, errors.MemoryError{}, errs[0])
 
- require.IsType(t, errors.MemoryError{}, panicMsg)
-
- fatalError, _ := panicMsg.(errors.MemoryError)
+ fatalError, _ := errs[0].(errors.MemoryError)
  var expectedError limitingMemoryGaugeError
  assert.ErrorAs(t, fatalError, &expectedError)
  })

runtime/repl.go

@@ -254,8 +254,11 @@ func (r *REPL) Accept(code []byte, eval bool) (inputIsComplete bool, err error)
  code = prefixedCode
  }
 
- tokens := lexer.Lex(code, nil)
+ tokens, err := lexer.Lex(code, nil)
  defer tokens.Reclaim()
+ if err != nil {
+ return
+ }
 
  inputIsComplete = isInputComplete(tokens)
 

runtime/runtime_test.go

@@ -7497,7 +7497,7 @@ func TestRuntimeInternalErrors(t *testing.T) {
 
  RequireError(t, err)
 
- assertRuntimeErrorIsInternalError(t, err)
+ assertRuntimeErrorIsExternalError(t, err)
  })
 
 }