From 681b7be5eed50e48673617b1819bdd0f4b05b793 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Tue, 13 Jun 2023 11:47:27 +0300 Subject: [PATCH 01/14] Add left recursion check --- Makefile | 4 +- ast/ast.go | 468 +++++++++++++++++++++++++++++++++ builder/builder.go | 26 +- builder/left_recursion.go | 134 ++++++++++ builder/left_recursion_test.go | 136 ++++++++++ builder/scc.go | 140 ++++++++++ builder/scc_test.go | 215 +++++++++++++++ go.mod | 6 + go.sum | 27 ++ main.go | 6 +- 10 files changed, 1154 insertions(+), 8 deletions(-) create mode 100644 builder/left_recursion.go create mode 100644 builder/left_recursion_test.go create mode 100644 builder/scc.go create mode 100644 builder/scc_test.go diff --git a/Makefile b/Makefile index ddf0f506..0b33214b 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ $(TEST_DIR)/goto_state/goto_state.go: $(TEST_DIR)/goto_state/goto_state.peg $(BI $(BINDIR)/pigeon -nolint $< > $@ $(TEST_DIR)/max_expr_cnt/maxexpr.go: $(TEST_DIR)/max_expr_cnt/maxexpr.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint $< > $@ + $(BINDIR)/pigeon -nolint -ignore-left-recursion $< > $@ $(TEST_DIR)/labeled_failures/labeled_failures.go: $(TEST_DIR)/labeled_failures/labeled_failures.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ @@ -167,7 +167,7 @@ $(TEST_DIR)/issue_70/optimized-grammar/issue_70.go: $(TEST_DIR)/issue_70/issue_7 $(BINDIR)/pigeon -nolint -optimize-grammar $< > $@ $(TEST_DIR)/issue_70b/issue_70b.go: $(TEST_DIR)/issue_70b/issue_70b.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint --optimize-grammar $< > $@ + $(BINDIR)/pigeon -nolint --optimize-grammar -ignore-left-recursion $< > $@ $(TEST_DIR)/issue_80/issue_80.go: $(TEST_DIR)/issue_80/issue_80.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ diff --git a/ast/ast.go b/ast/ast.go index c34b7d17..6302fcc0 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -35,6 +35,8 @@ type Grammar struct { Rules []*Rule } +var _ Expression = (*Grammar)(nil) + // NewGrammar creates a new grammar at the specified position. func NewGrammar(p Pos) *Grammar { return &Grammar{p: p} @@ -56,6 +58,21 @@ func (g *Grammar) String() string { return buf.String() } +// NullableVisit recursively determines whether an object is nullable +func (g *Grammar) NullableVisit(rules map[string]*Rule) bool { + panic("NullableVisit should not be called on the Grammar") +} + +// IsNullable returns the nullable attribute of the node +func (g *Grammar) IsNullable() bool { + panic("IsNullable should not be called on the Grammar") +} + +// InitialNames returns names of nodes with which an expression can begin +func (g *Grammar) InitialNames() map[string]bool { + panic("InitialNames should not be called on the Grammar") +} + // Rule represents a rule in the PEG grammar. It has a name, an optional // display name to be used in error messages, and an expression. type Rule struct { @@ -63,8 +80,16 @@ type Rule struct { Name *Identifier DisplayName *StringLit Expr Expression + + // for work with left recursion + Visited bool + Nullable bool + LeftRecursive bool + Leader bool } +var _ Expression = (*Rule)(nil) + // NewRule creates a rule with at the specified position and with the // specified name as identifier. func NewRule(p Pos, name *Identifier) *Rule { @@ -80,9 +105,35 @@ func (r *Rule) String() string { r.p, r, r.Name, r.DisplayName, r.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (r *Rule) NullableVisit(rules map[string]*Rule) bool { + if r.Visited { + // A left-recursive rule is considered non-nullable. + return false + } + r.Visited = true + r.Nullable = r.Expr.NullableVisit(rules) + return r.Nullable +} + +// IsNullable returns the nullable attribute of the node +func (r *Rule) IsNullable() bool { + return r.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (r *Rule) InitialNames() map[string]bool { + return r.Expr.InitialNames() +} + // Expression is the interface implemented by all expression types. type Expression interface { Pos() Pos + + // for work with left recursion + NullableVisit(rules map[string]*Rule) bool + IsNullable() bool + InitialNames() map[string]bool } // ChoiceExpr is an ordered sequence of expressions. The parser tries to @@ -91,8 +142,12 @@ type Expression interface { type ChoiceExpr struct { p Pos Alternatives []Expression + + Nullable bool } +var _ Expression = (*ChoiceExpr)(nil) + // NewChoiceExpr creates a choice expression at the specified position. func NewChoiceExpr(p Pos) *ChoiceExpr { return &ChoiceExpr{p: p} @@ -113,6 +168,34 @@ func (c *ChoiceExpr) String() string { return buf.String() } +// NullableVisit recursively determines whether an object is nullable +func (c *ChoiceExpr) NullableVisit(rules map[string]*Rule) bool { + for _, alt := range c.Alternatives { + if alt.NullableVisit(rules) { + c.Nullable = true + return true + } + } + c.Nullable = false + return false +} + +// IsNullable returns the nullable attribute of the node +func (c *ChoiceExpr) IsNullable() bool { + return c.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (c *ChoiceExpr) InitialNames() map[string]bool { + names := make(map[string]bool) + for _, alt := range c.Alternatives { + for name := range alt.InitialNames() { + names[name] = true + } + } + return names +} + // FailureLabel is an identifier, which can by thrown and recovered in a grammar type FailureLabel string @@ -124,8 +207,12 @@ type RecoveryExpr struct { Expr Expression RecoverExpr Expression Labels []FailureLabel + + Nullable bool } +var _ Expression = (*RecoveryExpr)(nil) + // NewRecoveryExpr creates a choice expression at the specified position. func NewRecoveryExpr(p Pos) *RecoveryExpr { return &RecoveryExpr{p: p} @@ -147,6 +234,29 @@ func (r *RecoveryExpr) String() string { return buf.String() } +// NullableVisit recursively determines whether an object is nullable +func (r *RecoveryExpr) NullableVisit(rules map[string]*Rule) bool { + r.Nullable = r.Expr.NullableVisit(rules) || r.RecoverExpr.NullableVisit(rules) + return r.Nullable +} + +// IsNullable returns the nullable attribute of the node +func (r *RecoveryExpr) IsNullable() bool { + return r.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (r *RecoveryExpr) InitialNames() map[string]bool { + names := make(map[string]bool) + for name := range r.Expr.InitialNames() { + names[name] = true + } + for name := range r.RecoverExpr.InitialNames() { + names[name] = true + } + return names +} + // ActionExpr is an expression that has an associated block of code to // execute when the expression matches. type ActionExpr struct { @@ -154,8 +264,12 @@ type ActionExpr struct { Expr Expression Code *CodeBlock FuncIx int + + Nullable bool } +var _ Expression = (*ActionExpr)(nil) + // NewActionExpr creates a new action expression at the specified position. func NewActionExpr(p Pos) *ActionExpr { return &ActionExpr{p: p} @@ -169,6 +283,26 @@ func (a *ActionExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v, Code: %v}", a.p, a, a.Expr, a.Code) } +// NullableVisit recursively determines whether an object is nullable +func (a *ActionExpr) NullableVisit(rules map[string]*Rule) bool { + a.Nullable = a.Expr.NullableVisit(rules) + return a.Nullable +} + +// IsNullable returns the nullable attribute of the node +func (a *ActionExpr) IsNullable() bool { + return a.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (a *ActionExpr) InitialNames() map[string]bool { + names := make(map[string]bool) + for name := range a.Expr.InitialNames() { + names[name] = true + } + return names +} + // ThrowExpr is an expression that throws an FailureLabel to be catched by a // RecoveryChoiceExpr. type ThrowExpr struct { @@ -176,6 +310,8 @@ type ThrowExpr struct { Label string } +var _ Expression = (*ThrowExpr)(nil) + // NewThrowExpr creates a new throw expression at the specified position. func NewThrowExpr(p Pos) *ThrowExpr { return &ThrowExpr{p: p} @@ -189,13 +325,32 @@ func (t *ThrowExpr) String() string { return fmt.Sprintf("%s: %T{Label: %v}", t.p, t, t.Label) } +// NullableVisit recursively determines whether an object is nullable +func (t *ThrowExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (t *ThrowExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (t *ThrowExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // SeqExpr is an ordered sequence of expressions, all of which must match // if the SeqExpr is to be a match itself. type SeqExpr struct { p Pos Exprs []Expression + + Nullable bool } +var _ Expression = (*SeqExpr)(nil) + // NewSeqExpr creates a new sequence expression at the specified position. func NewSeqExpr(p Pos) *SeqExpr { return &SeqExpr{p: p} @@ -216,6 +371,37 @@ func (s *SeqExpr) String() string { return buf.String() } +// NullableVisit recursively determines whether an object is nullable +func (s *SeqExpr) NullableVisit(rules map[string]*Rule) bool { + for _, item := range s.Exprs { + if !item.NullableVisit(rules) { + s.Nullable = false + return false + } + } + s.Nullable = true + return true +} + +// IsNullable returns the nullable attribute of the node +func (s *SeqExpr) IsNullable() bool { + return s.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (s *SeqExpr) InitialNames() map[string]bool { + names := make(map[string]bool) + for _, item := range s.Exprs { + for name := range item.InitialNames() { + names[name] = true + } + if !item.IsNullable() { + break + } + } + return names +} + // LabeledExpr is an expression that has an associated label. Code blocks // can access the value of the expression using that label, that becomes // a local variable in the code. @@ -225,6 +411,8 @@ type LabeledExpr struct { Expr Expression } +var _ Expression = (*LabeledExpr)(nil) + // NewLabeledExpr creates a new labeled expression at the specified position. func NewLabeledExpr(p Pos) *LabeledExpr { return &LabeledExpr{p: p} @@ -238,6 +426,21 @@ func (l *LabeledExpr) String() string { return fmt.Sprintf("%s: %T{Label: %v, Expr: %v}", l.p, l, l.Label, l.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (l *LabeledExpr) NullableVisit(rules map[string]*Rule) bool { + return l.Expr.NullableVisit(rules) +} + +// IsNullable returns the nullable attribute of the node +func (l *LabeledExpr) IsNullable() bool { + return l.Expr.IsNullable() +} + +// InitialNames returns names of nodes with which an expression can begin +func (l *LabeledExpr) InitialNames() map[string]bool { + return l.Expr.InitialNames() +} + // AndExpr is a zero-length matcher that is considered a match if the // expression it contains is a match. type AndExpr struct { @@ -250,6 +453,8 @@ func NewAndExpr(p Pos) *AndExpr { return &AndExpr{p: p} } +var _ Expression = (*AndExpr)(nil) + // Pos returns the starting position of the node. func (a *AndExpr) Pos() Pos { return a.p } @@ -258,6 +463,21 @@ func (a *AndExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", a.p, a, a.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (a *AndExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (a *AndExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (a *AndExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // NotExpr is a zero-length matcher that is considered a match if the // expression it contains is not a match. type NotExpr struct { @@ -265,6 +485,8 @@ type NotExpr struct { Expr Expression } +var _ Expression = (*NotExpr)(nil) + // NewNotExpr creates a new not (!) expression at the specified position. func NewNotExpr(p Pos) *NotExpr { return &NotExpr{p: p} @@ -278,12 +500,29 @@ func (n *NotExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", n.p, n, n.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (n *NotExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (n *NotExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (n *NotExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // ZeroOrOneExpr is an expression that can be matched zero or one time. type ZeroOrOneExpr struct { p Pos Expr Expression } +var _ Expression = (*ZeroOrOneExpr)(nil) + // NewZeroOrOneExpr creates a new zero or one expression at the specified // position. func NewZeroOrOneExpr(p Pos) *ZeroOrOneExpr { @@ -298,12 +537,29 @@ func (z *ZeroOrOneExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (z *ZeroOrOneExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (z *ZeroOrOneExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (z *ZeroOrOneExpr) InitialNames() map[string]bool { + return z.Expr.InitialNames() +} + // ZeroOrMoreExpr is an expression that can be matched zero or more times. type ZeroOrMoreExpr struct { p Pos Expr Expression } +var _ Expression = (*ZeroOrMoreExpr)(nil) + // NewZeroOrMoreExpr creates a new zero or more expression at the specified // position. func NewZeroOrMoreExpr(p Pos) *ZeroOrMoreExpr { @@ -318,12 +574,29 @@ func (z *ZeroOrMoreExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (z *ZeroOrMoreExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (z *ZeroOrMoreExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (z *ZeroOrMoreExpr) InitialNames() map[string]bool { + return z.Expr.InitialNames() +} + // OneOrMoreExpr is an expression that can be matched one or more times. type OneOrMoreExpr struct { p Pos Expr Expression } +var _ Expression = (*OneOrMoreExpr)(nil) + // NewOneOrMoreExpr creates a new one or more expression at the specified // position. func NewOneOrMoreExpr(p Pos) *OneOrMoreExpr { @@ -338,12 +611,31 @@ func (o *OneOrMoreExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", o.p, o, o.Expr) } +// NullableVisit recursively determines whether an object is nullable +func (o *OneOrMoreExpr) NullableVisit(rules map[string]*Rule) bool { + return false +} + +// IsNullable returns the nullable attribute of the node +func (o *OneOrMoreExpr) IsNullable() bool { + return false +} + +// InitialNames returns names of nodes with which an expression can begin +func (o *OneOrMoreExpr) InitialNames() map[string]bool { + return o.Expr.InitialNames() +} + // RuleRefExpr is an expression that references a rule by name. type RuleRefExpr struct { p Pos Name *Identifier + + Nullable bool } +var _ Expression = (*RuleRefExpr)(nil) + // NewRuleRefExpr creates a new rule reference expression at the specified // position. func NewRuleRefExpr(p Pos) *RuleRefExpr { @@ -358,6 +650,28 @@ func (r *RuleRefExpr) String() string { return fmt.Sprintf("%s: %T{Name: %v}", r.p, r, r.Name) } +// NullableVisit recursively determines whether an object is nullable +func (r *RuleRefExpr) NullableVisit(rules map[string]*Rule) bool { + item, ok := rules[r.Name.Val] + if !ok { + // Token or unknown; never empty. + r.Nullable = false + return false + } + r.Nullable = item.NullableVisit(rules) + return r.Nullable +} + +// IsNullable returns the nullable attribute of the node +func (r *RuleRefExpr) IsNullable() bool { + return r.Nullable +} + +// InitialNames returns names of nodes with which an expression can begin +func (r *RuleRefExpr) InitialNames() map[string]bool { + return map[string]bool{r.Name.Val: true} +} + // StateCodeExpr is an expression which can modify the internal state of the parser. type StateCodeExpr struct { p Pos @@ -365,6 +679,8 @@ type StateCodeExpr struct { FuncIx int } +var _ Expression = (*StateCodeExpr)(nil) + // NewStateCodeExpr creates a new state (#) code expression at the specified // position. func NewStateCodeExpr(p Pos) *StateCodeExpr { @@ -379,6 +695,21 @@ func (s *StateCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", s.p, s, s.Code) } +// NullableVisit recursively determines whether an object is nullable +func (s *StateCodeExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (s *StateCodeExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (s *StateCodeExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // AndCodeExpr is a zero-length matcher that is considered a match if the // code block returns true. type AndCodeExpr struct { @@ -387,6 +718,8 @@ type AndCodeExpr struct { FuncIx int } +var _ Expression = (*AndCodeExpr)(nil) + // NewAndCodeExpr creates a new and (&) code expression at the specified // position. func NewAndCodeExpr(p Pos) *AndCodeExpr { @@ -401,6 +734,21 @@ func (a *AndCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", a.p, a, a.Code) } +// NullableVisit recursively determines whether an object is nullable +func (a *AndCodeExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (a *AndCodeExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (a *AndCodeExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // NotCodeExpr is a zero-length matcher that is considered a match if the // code block returns false. type NotCodeExpr struct { @@ -409,6 +757,8 @@ type NotCodeExpr struct { FuncIx int } +var _ Expression = (*NotCodeExpr)(nil) + // NewNotCodeExpr creates a new not (!) code expression at the specified // position. func NewNotCodeExpr(p Pos) *NotCodeExpr { @@ -423,6 +773,21 @@ func (n *NotCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", n.p, n, n.Code) } +// NullableVisit recursively determines whether an object is nullable +func (n *NotCodeExpr) NullableVisit(rules map[string]*Rule) bool { + return true +} + +// IsNullable returns the nullable attribute of the node +func (n *NotCodeExpr) IsNullable() bool { + return true +} + +// InitialNames returns names of nodes with which an expression can begin +func (n *NotCodeExpr) InitialNames() map[string]bool { + return make(map[string]bool) +} + // LitMatcher is a string literal matcher. The value to match may be a // double-quoted string, a single-quoted single character, or a back-tick // quoted raw string. @@ -431,6 +796,8 @@ type LitMatcher struct { IgnoreCase bool } +var _ Expression = (*LitMatcher)(nil) + // NewLitMatcher creates a new literal matcher at the specified position and // with the specified value. func NewLitMatcher(p Pos, v string) *LitMatcher { @@ -445,6 +812,22 @@ func (l *LitMatcher) String() string { return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t}", l.p, l, l.Val, l.IgnoreCase) } +// NullableVisit recursively determines whether an object is nullable +func (l *LitMatcher) NullableVisit(rules map[string]*Rule) bool { + return l.IsNullable() +} + +// IsNullable returns the nullable attribute of the node +func (l *LitMatcher) IsNullable() bool { + // The string token '' is considered empty. + return len(l.Val) == 0 +} + +// InitialNames returns names of nodes with which an expression can begin +func (l *LitMatcher) InitialNames() map[string]bool { + return make(map[string]bool) +} + // CharClassMatcher is a character class matcher. The value to match must // be one of the specified characters, in a range of characters, or in the // Unicode classes of characters. @@ -457,6 +840,8 @@ type CharClassMatcher struct { UnicodeClasses []string } +var _ Expression = (*CharClassMatcher)(nil) + // NewCharClassMatcher creates a new character class matcher at the specified // position and with the specified raw value. It parses the raw value into // the list of characters, ranges and Unicode classes. @@ -580,11 +965,28 @@ func (c *CharClassMatcher) String() string { c.p, c, c.Val, c.IgnoreCase, c.Inverted) } +// NullableVisit recursively determines whether an object is nullable +func (c *CharClassMatcher) NullableVisit(rules map[string]*Rule) bool { + return c.IsNullable() +} + +// IsNullable returns the nullable attribute of the node +func (c *CharClassMatcher) IsNullable() bool { + return len(c.Chars) == 0 && len(c.Ranges) == 0 && len(c.UnicodeClasses) == 0 +} + +// InitialNames returns names of nodes with which an expression can begin +func (c *CharClassMatcher) InitialNames() map[string]bool { + return make(map[string]bool) +} + // AnyMatcher is a matcher that matches any character except end-of-file. type AnyMatcher struct { posValue } +var _ Expression = (*AnyMatcher)(nil) + // NewAnyMatcher creates a new any matcher at the specified position. The // value is provided for completeness' sake, but it is always the dot. func NewAnyMatcher(p Pos, v string) *AnyMatcher { @@ -599,11 +1001,28 @@ func (a *AnyMatcher) String() string { return fmt.Sprintf("%s: %T{Val: %q}", a.p, a, a.Val) } +// NullableVisit recursively determines whether an object is nullable +func (a *AnyMatcher) NullableVisit(rules map[string]*Rule) bool { + return false +} + +// IsNullable returns the nullable attribute of the node +func (a *AnyMatcher) IsNullable() bool { + return false +} + +// InitialNames returns names of nodes with which an expression can begin +func (a *AnyMatcher) InitialNames() map[string]bool { + return make(map[string]bool) +} + // CodeBlock represents a code block. type CodeBlock struct { posValue } +var _ Expression = (*CodeBlock)(nil) + // NewCodeBlock creates a new code block at the specified position and with // the specified value. The value includes the outer braces. func NewCodeBlock(p Pos, code string) *CodeBlock { @@ -618,11 +1037,28 @@ func (c *CodeBlock) String() string { return fmt.Sprintf("%s: %T{Val: %q}", c.p, c, c.Val) } +// NullableVisit recursively determines whether an object is nullable +func (c *CodeBlock) NullableVisit(rules map[string]*Rule) bool { + panic("NullableVisit should not be called on the CodeBlock") +} + +// IsNullable returns the nullable attribute of the node +func (c *CodeBlock) IsNullable() bool { + panic("IsNullable should not be called on the CodeBlock") +} + +// InitialNames returns names of nodes with which an expression can begin +func (c *CodeBlock) InitialNames() map[string]bool { + panic("InitialNames should not be called on the CodeBlock") +} + // Identifier represents an identifier. type Identifier struct { posValue } +var _ Expression = (*Identifier)(nil) + // NewIdentifier creates a new identifier at the specified position and // with the specified name. func NewIdentifier(p Pos, name string) *Identifier { @@ -637,11 +1073,28 @@ func (i *Identifier) String() string { return fmt.Sprintf("%s: %T{Val: %q}", i.p, i, i.Val) } +// NullableVisit recursively determines whether an object is nullable +func (i *Identifier) NullableVisit(rules map[string]*Rule) bool { + panic("NullableVisit should not be called on the Identifier") +} + +// IsNullable returns the nullable attribute of the node +func (i *Identifier) IsNullable() bool { + panic("IsNullable should not be called on the Identifier") +} + +// InitialNames returns names of nodes with which an expression can begin +func (i *Identifier) InitialNames() map[string]bool { + panic("InitialNames should not be called on the Identifier") +} + // StringLit represents a string literal. type StringLit struct { posValue } +var _ Expression = (*StringLit)(nil) + // NewStringLit creates a new string literal at the specified position and // with the specified value. func NewStringLit(p Pos, val string) *StringLit { @@ -656,6 +1109,21 @@ func (s *StringLit) String() string { return fmt.Sprintf("%s: %T{Val: %q}", s.p, s, s.Val) } +// NullableVisit recursively determines whether an object is nullable +func (s *StringLit) NullableVisit(rules map[string]*Rule) bool { + panic("NullableVisit should not be called on the StringLit") +} + +// IsNullable returns the nullable attribute of the node +func (s *StringLit) IsNullable() bool { + panic("IsNullable should not be called on the StringLit") +} + +// InitialNames returns names of nodes with which an expression can begin +func (s *StringLit) InitialNames() map[string]bool { + panic("InitialNames should not be called on the StringLit") +} + type posValue struct { p Pos Val string diff --git a/builder/builder.go b/builder/builder.go index ee76666a..9d067327 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -77,6 +77,17 @@ func Optimize(optimize bool) Option { } } +// IgnoreLeftRecursion returns an option that specifies the ignoreLeftRecursion option +// If ignoreLeftRecursion is true, errors associated +// with the use of left recursion rules are ignored. +func IgnoreLeftRecursion(ignore bool) Option { + return func(b *builder) Option { + prev := b.optimize + b.ignoreLeftRecursion = ignore + return IgnoreLeftRecursion(prev) + } +} + // Nolint returns an option that specifies the nolint option // If nolint is true, special '// nolint: ...' comments are added // to the generated parser to suppress warnings by gometalinter. @@ -118,6 +129,7 @@ type builder struct { basicLatinLookupTable bool globalState bool nolint bool + ignoreLeftRecursion bool ruleName string exprIndex int @@ -132,11 +144,15 @@ func (b *builder) setOptions(opts []Option) { } } -func (b *builder) buildParser(g *ast.Grammar) error { - b.writeInit(g.Init) - b.writeGrammar(g) - - for _, rule := range g.Rules { +func (b *builder) buildParser(grammar *ast.Grammar) error { + if err := PrepareGramma(grammar); err != nil { + if !b.ignoreLeftRecursion { + return fmt.Errorf("uncorrect gramma: %w", err) + } + } + b.writeInit(grammar.Init) + b.writeGrammar(grammar) + for _, rule := range grammar.Rules { b.writeRuleCode(rule) } b.writeStaticCode() diff --git a/builder/left_recursion.go b/builder/left_recursion.go new file mode 100644 index 00000000..a93c8b26 --- /dev/null +++ b/builder/left_recursion.go @@ -0,0 +1,134 @@ +package builder + +import ( + "errors" + "fmt" + + "github.com/mna/pigeon/ast" +) + +var ( + // ErrNoLeader is no leader error. + ErrNoLeader = errors.New( + "SCC has no leadership candidate (no element is included in all cycles)") + // ErrHaveLeftRecirsion is recursion error. + ErrHaveLeftRecirsion = errors.New("have left recursion") +) + +// PrepareGramma evaluates parameters associated with left recursion +func PrepareGramma(grammar *ast.Grammar) error { + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { + mapRules[rule.Name.Val] = rule + } + ComputeNullables(mapRules) + if err := ComputeLeftRecursives(mapRules); err != nil { + return fmt.Errorf("error compute left recursive: %w", err) + } + rulesWithLeftRecursion := []string{} + for _, rule := range grammar.Rules { + if rule.LeftRecursive { + rulesWithLeftRecursion = append(rulesWithLeftRecursion, rule.Name.Val) + } + } + if len(rulesWithLeftRecursion) > 0 { + return fmt.Errorf("%w: %v", ErrHaveLeftRecirsion, rulesWithLeftRecursion) + } + + return nil +} + +// ComputeNullables evaluates nullable nodes +func ComputeNullables(rules map[string]*ast.Rule) { + // Compute which rules in a grammar are nullable + for _, rule := range rules { + rule.NullableVisit(rules) + } +} + +func findLeader( + graph map[string]map[string]bool, scc map[string]bool, +) (string, error) { + // Try to find a leader such that all cycles go through it. + leaders := make(map[string]bool, len(scc)) + for k := range scc { + leaders[k] = true + } + for start := range scc { + for _, cycle := range FindCyclesInSCC(graph, scc, start) { + mapCycle := map[string]bool{} + for _, k := range cycle { + mapCycle[k] = true + } + for k := range scc { + if _, okCycle := mapCycle[k]; !okCycle { + delete(leaders, k) + } + } + if len(leaders) == 0 { + return "", ErrNoLeader + } + } + } + // Pick an arbitrary leader from the candidates. + var leader string + for k := range leaders { + leader = k // The only element. + break + } + return leader, nil +} + +// ComputeLeftRecursives evaluates left recursion +func ComputeLeftRecursives(rules map[string]*ast.Rule) error { + graph := MakeFirstGraph(rules) + vertices := make([]string, 0, len(graph)) + for k := range graph { + vertices = append(vertices, k) + } + sccs := StronglyConnectedComponents(vertices, graph) + for _, scc := range sccs { + if len(scc) > 1 { + for name := range scc { + rules[name].LeftRecursive = true + } + leader, err := findLeader(graph, scc) + if err != nil { + return fmt.Errorf("error find leader %v: %w", scc, err) + } + rules[leader].Leader = true + } else { + var name string + for k := range scc { + name = k // The only element. + break + } + if _, ok := graph[name][name]; ok { + rules[name].LeftRecursive = true + rules[name].Leader = true + } + } + } + return nil +} + +// MakeFirstGraph compute the graph of left-invocations. +// There's an edge from A to B if A may invoke B at its initial position. +// Note that this requires the nullable flags to have been computed. +func MakeFirstGraph(rules map[string]*ast.Rule) map[string]map[string]bool { + graph := make(map[string]map[string]bool) + vertices := make(map[string]bool) + for rulename, rule := range rules { + names := rule.InitialNames() + graph[rulename] = names + for name := range names { + vertices[name] = true + } + } + for vertex := range vertices { + if _, ok := graph[vertex]; !ok { + graph[vertex] = make(map[string]bool) + } + } + return graph +} diff --git a/builder/left_recursion_test.go b/builder/left_recursion_test.go new file mode 100644 index 00000000..64aad87b --- /dev/null +++ b/builder/left_recursion_test.go @@ -0,0 +1,136 @@ +package builder_test + +import ( + "strings" + "testing" + + "github.com/mna/pigeon/ast" + "github.com/mna/pigeon/bootstrap" + "github.com/mna/pigeon/builder" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLeftRrecursive(t *testing.T) { + t.Parallel() + grammar := ` + start = expr NEWLINE + expr = ('-' term / expr '+' term / term) + term = NUMBER + foo = NAME+ + bar = NAME* + baz = NAME? + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) + mapRules := make(map[string]*ast.Rule, len(g.Rules)) + for _, rule := range g.Rules { + mapRules[rule.Name.Val] = rule + } + assert.False(t, mapRules["start"].LeftRecursive) + assert.True(t, mapRules["expr"].LeftRecursive) + assert.False(t, mapRules["term"].LeftRecursive) + assert.False(t, mapRules["foo"].LeftRecursive) + assert.False(t, mapRules["bar"].LeftRecursive) + assert.False(t, mapRules["baz"].LeftRecursive) +} + +func TestNullable(t *testing.T) { + t.Parallel() + grammar := ` + start = sign NUMBER + sign = ('-' / '+')? + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.NoError(t, err) + mapRules := make(map[string]*ast.Rule, len(g.Rules)) + for _, rule := range g.Rules { + mapRules[rule.Name.Val] = rule + } + assert.False(t, mapRules["start"].Nullable) + assert.True(t, mapRules["sign"].Nullable) +} + +func TestAdvancedLeftRrecursive(t *testing.T) { + t.Parallel() + grammar := ` + start = NUMBER / sign start + sign = '-'? + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) + mapRules := make(map[string]*ast.Rule, len(g.Rules)) + for _, rule := range g.Rules { + mapRules[rule.Name.Val] = rule + } + assert.False(t, mapRules["start"].Nullable) + assert.True(t, mapRules["sign"].Nullable) + assert.True(t, mapRules["start"].LeftRecursive) + assert.False(t, mapRules["sign"].LeftRecursive) +} + +func TestMutuallyLeftRrecursive(t *testing.T) { + t.Parallel() + grammar := ` + start = foo 'E' + foo = bar 'A' / 'B' + bar = foo 'C' / 'D' + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) + mapRules := make(map[string]*ast.Rule, len(g.Rules)) + for _, rule := range g.Rules { + mapRules[rule.Name.Val] = rule + } + assert.False(t, mapRules["start"].LeftRecursive) + assert.True(t, mapRules["foo"].LeftRecursive) + assert.True(t, mapRules["bar"].LeftRecursive) +} + +func TestNastyMutuallyLeftRrecursive(t *testing.T) { + t.Parallel() + grammar := ` + start = target '=' + target = maybe '+' / NAME + maybe = maybe '-' / target + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) + mapRules := make(map[string]*ast.Rule, len(g.Rules)) + for _, rule := range g.Rules { + mapRules[rule.Name.Val] = rule + } + assert.False(t, mapRules["start"].LeftRecursive) + assert.True(t, mapRules["target"].LeftRecursive) + assert.True(t, mapRules["maybe"].LeftRecursive) +} + +func TestLeftRecursionTooComplex(t *testing.T) { + t.Parallel() + grammar := ` + start = foo + foo = bar '+' / baz '+' / '+' + bar = baz '-' / foo '-' / '-' + baz = foo '*' / bar '*' / '*' + ` + p := bootstrap.NewParser() + g, err := p.Parse("", strings.NewReader(grammar)) + require.NoError(t, err) + err = builder.PrepareGramma(g) + require.ErrorIs(t, err, builder.ErrNoLeader) +} diff --git a/builder/scc.go b/builder/scc.go new file mode 100644 index 00000000..2de71eae --- /dev/null +++ b/builder/scc.go @@ -0,0 +1,140 @@ +package builder + +import "fmt" + +func min(a1 int, a2 int) int { + if a1 <= a2 { + return a1 + } + return a2 +} + +// StronglyConnectedComponents compute strongly сonnected сomponents of a graph. +// Tarjan's strongly connected components algorithm +func StronglyConnectedComponents( + vertices []string, edges map[string]map[string]bool, +) []map[string]bool { + // Tarjan's strongly connected components algorithm + var ( + identified = map[string]bool{} + stack = []string{} + index = map[string]int{} + lowlink = map[string]int{} + dfs func(v string) []map[string]bool + ) + + dfs = func(vertex string) []map[string]bool { + index[vertex] = len(stack) + stack = append(stack, vertex) + lowlink[vertex] = index[vertex] + + sccs := []map[string]bool{} + for w := range edges[vertex] { + if _, ok := index[w]; !ok { + sccs = append(sccs, dfs(w)...) + lowlink[vertex] = min(lowlink[vertex], lowlink[w]) + } else if _, ok := identified[w]; !ok { + lowlink[vertex] = min(lowlink[vertex], lowlink[w]) + } + } + + if lowlink[vertex] == index[vertex] { + scc := map[string]bool{} + for _, v := range stack[index[vertex]:] { + scc[v] = true + } + stack = stack[:index[vertex]] + for v := range scc { + identified[v] = true + } + sccs = append(sccs, scc) + } + return sccs + } + + sccs := []map[string]bool{} + for _, v := range vertices { + if _, ok := index[v]; !ok { + sccs = append(sccs, dfs(v)...) + } + } + return sccs +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +func reduceGraph( + graph map[string]map[string]bool, scc map[string]bool, +) map[string]map[string]bool { + reduceGraph := map[string]map[string]bool{} + for src, dsts := range graph { + if _, ok := scc[src]; !ok { + continue + } + reduceGraph[src] = map[string]bool{} + for dst := range dsts { + if _, ok := scc[dst]; !ok { + continue + } + reduceGraph[src][dst] = true + } + } + return reduceGraph +} + +// FindCyclesInSCC find cycles in SCC emanating from start. +// Yields lists of the form ['A', 'B', 'C', 'A'], which means there's +// a path from A -> B -> C -> A. The first item is always the start +// argument, but the last item may be another element, e.g. ['A', +// 'B', 'C', 'B'] means there's a path from A to B and there's a +// cycle from B to C and back. +func FindCyclesInSCC( + graph map[string]map[string]bool, scc map[string]bool, start string, +) [][]string { + // Basic input checks. + if _, ok := scc[start]; !ok { + panic(fmt.Sprintf("scc %v have not %v", scc, start)) + } + extravertices := []string{} + for k := range scc { + if _, ok := graph[k]; !ok { + extravertices = append(extravertices, k) + } + } + if len(extravertices) != 0 { + panic(fmt.Sprintf("graph have not scc. %v", extravertices)) + } + + // Reduce the graph to nodes in the SCC. + graph = reduceGraph(graph, scc) + if _, ok := graph[start]; !ok { + panic(fmt.Sprintf("graph %v have not %v", graph, start)) + } + + // Recursive helper that yields cycles. + var dfs func(node string, path []string) [][]string + dfs = func(node string, path []string) [][]string { + ret := [][]string{} + if contains(path, node) { + t := make([]string, 0, len(path)+1) + t = append(t, path...) + t = append(t, node) + ret = append(ret, t) + return ret + } + path = append(path, node) // TODO: Make this not quadratic. + for child := range graph[node] { + ret = append(ret, dfs(child, path)...) + } + return ret + } + + return dfs(start, []string{}) +} diff --git a/builder/scc_test.go b/builder/scc_test.go new file mode 100644 index 00000000..71161434 --- /dev/null +++ b/builder/scc_test.go @@ -0,0 +1,215 @@ +package builder_test + +import ( + "testing" + + "github.com/mna/pigeon/builder" + "github.com/stretchr/testify/require" +) + +func TestStronglyConnectedComponents(t *testing.T) { //nolint:funlen + t.Parallel() + + type want struct { + sccs []map[string]bool + } + + tests := []struct { + name string + graph map[string]map[string]bool + want want + }{ + { + name: "Simple", + graph: map[string]map[string]bool{ + "1": {"2": true}, + "2": {"1": true}, + }, + want: want{sccs: []map[string]bool{ + {"2": true, "1": true}, + }}, + }, + { + name: "Without scc", + graph: map[string]map[string]bool{ + "1": {"2": true}, + }, + want: want{sccs: []map[string]bool{ + {"2": true}, + {"1": true}, + }}, + }, + { + name: "One element", + graph: map[string]map[string]bool{ + "1": {}, + }, + want: want{sccs: []map[string]bool{ + {"1": true}, + }}, + }, + { + name: "One element with loop", + graph: map[string]map[string]bool{ + "1": {"1": true}, + }, + want: want{sccs: []map[string]bool{ + {"1": true}, + }}, + }, + { + name: "Wiki 1", + graph: map[string]map[string]bool{ + "1": {"2": true}, + "2": {"3": true}, + "3": {"1": true}, + "4": {"2": true, "3": true, "6": true}, + "5": {"3": true, "7": true}, + "6": {"4": true, "5": true}, + "7": {"5": true}, + "8": {"6": true, "7": true, "8": true}, + }, + want: want{sccs: []map[string]bool{ + {"2": true, "3": true, "1": true}, + {"5": true, "7": true}, + {"4": true, "6": true}, + {"8": true}, + }}, + }, + { + name: "Wiki 2", + graph: map[string]map[string]bool{ + "1": {"2": true, "6": true}, + "2": {"6": true, "4": true}, + "3": {"9": true, "4": true, "8": true}, + "4": {"1": true, "7": true}, + "5": {"9": true, "8": true}, + "6": {"1": true, "4": true, "7": true}, + "7": {"1": true}, + "8": {"5": true, "3": true}, + "9": {"8": true}, + }, + want: want{sccs: []map[string]bool{ + {"1": true, "2": true, "4": true, "6": true, "7": true}, + {"3": true, "5": true, "9": true, "8": true}, + }}, + }, + } + + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + vertices := make([]string, 0, len(testCase.graph)) + for k := range testCase.graph { + vertices = append(vertices, k) + } + require.ElementsMatch(t, builder.StronglyConnectedComponents( + vertices, testCase.graph), testCase.want.sccs) + }) + } +} + +func TestFindCyclesInSCC(t *testing.T) { //nolint:funlen + t.Parallel() + + type want struct { + paths [][]string + } + + tests := []struct { + name string + graph map[string]map[string]bool + scc map[string]bool + start string + want want + }{ + { + name: "Wiki 1 1", + graph: map[string]map[string]bool{ + "1": {"2": true}, + "2": {"3": true}, + "3": {"1": true}, + "4": {"2": true, "3": true, "6": true}, + "5": {"3": true, "7": true}, + "6": {"4": true, "5": true}, + "7": {"5": true}, + "8": {"6": true, "7": true, "8": true}, + }, + scc: map[string]bool{"2": true, "3": true, "1": true}, + start: "3", + want: want{paths: [][]string{{"3", "1", "2", "3"}}}, + }, + { + name: "Wiki 1 2", + graph: map[string]map[string]bool{ + "1": {"2": true}, + "2": {"3": true}, + "3": {"1": true}, + "4": {"2": true, "3": true, "6": true}, + "5": {"3": true, "7": true}, + "6": {"4": true, "5": true}, + "7": {"5": true}, + "8": {"6": true, "7": true, "8": true}, + }, + scc: map[string]bool{"5": true, "7": true}, + start: "5", + want: want{paths: [][]string{{"5", "7", "5"}}}, + }, + { + name: "Wiki 2", + graph: map[string]map[string]bool{ + "1": {"2": true, "6": true}, + "2": {"6": true, "4": true}, + "3": {"9": true, "4": true, "8": true}, + "4": {"1": true, "7": true}, + "5": {"9": true, "8": true}, + "6": {"1": true, "4": true, "7": true}, + "7": {"1": true}, + "8": {"5": true, "3": true}, + "9": {"8": true}, + }, + scc: map[string]bool{ + "1": true, "2": true, "4": true, "6": true, "7": true, + }, + start: "1", + want: want{paths: [][]string{ + {"1", "2", "6", "1"}, + {"1", "2", "6", "4", "1"}, + {"1", "2", "6", "4", "7", "1"}, + {"1", "2", "6", "7", "1"}, + {"1", "2", "4", "1"}, + {"1", "2", "4", "7", "1"}, + {"1", "6", "1"}, + {"1", "6", "7", "1"}, + {"1", "6", "4", "7", "1"}, + {"1", "6", "4", "1"}, + }}, + }, + { + name: "loop in loop", + graph: map[string]map[string]bool{ + "1": {"2": true}, + "2": {"3": true}, + "3": {"1": true, "2": true}, + }, + scc: map[string]bool{ + "1": true, "2": true, "3": true, + }, + start: "1", + want: want{paths: [][]string{ + {"1", "2", "3", "1"}, + {"1", "2", "3", "2"}, + }}, + }, + } + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + require.ElementsMatch(t, builder.FindCyclesInSCC( + testCase.graph, testCase.scc, testCase.start), + testCase.want.paths) + }) + } +} diff --git a/go.mod b/go.mod index b260a3d0..5cdce67a 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,10 @@ require golang.org/x/tools v0.10.0 require ( golang.org/x/mod v0.11.0 // indirect golang.org/x/sys v0.9.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1b85af0a..3fe54c21 100644 --- a/go.sum +++ b/go.sum @@ -4,3 +4,30 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= + diff --git a/main.go b/main.go index e1d3423c..8f13bf77 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,7 @@ func main() { outputFlag = fs.String("o", "", "output file, defaults to stdout") optimizeBasicLatinFlag = fs.Bool("optimize-basic-latin", false, "generate optimized parser for Unicode Basic Latin character sets") optimizeGrammar = fs.Bool("optimize-grammar", false, "optimize the given grammar (EXPERIMENTAL FEATURE)") + ignoreLeftRecursion = fs.Bool("ignore-left-recursion", false, "ignore errors related to left recursion") optimizeParserFlag = fs.Bool("optimize-parser", false, "generate optimized parser without Debug and Memoize options") recvrNmFlag = fs.String("receiver-name", "c", "receiver name for the generated methods") noBuildFlag = fs.Bool("x", false, "do not build, only parse") @@ -136,7 +137,10 @@ func main() { optimizeParser := builder.Optimize(*optimizeParserFlag) basicLatinOptimize := builder.BasicLatinLookupTable(*optimizeBasicLatinFlag) nolintOpt := builder.Nolint(*nolint) - if err := builder.BuildParser(outBuf, grammar, curNmOpt, optimizeParser, basicLatinOptimize, nolintOpt); err != nil { + leftRecursionIgnorer := builder.IgnoreLeftRecursion(*ignoreLeftRecursion) + if err := builder.BuildParser( + outBuf, grammar, curNmOpt, optimizeParser, basicLatinOptimize, + nolintOpt, leftRecursionIgnorer); err != nil { fmt.Fprintln(os.Stderr, "build error: ", err) exit(5) } From 040bfe9bbf57c8323a2cff3416e440eead61bdde Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Thu, 15 Jun 2023 17:15:56 +0300 Subject: [PATCH 02/14] Add support left recursion --- Makefile | 7 +- .../cmd/bootstrap-pigeon/bootstrap_pigeon.go | 216 +- builder/builder.go | 30 +- builder/generated_static_code.go | 199 +- builder/left_recursion.go | 30 +- builder/left_recursion_test.go | 69 +- builder/static_code.go | 199 +- examples/calculator/calculator.go | 120 +- examples/indentation/indentation.go | 140 +- examples/json/json.go | 140 +- examples/json/optimized-grammar/json.go | 110 +- examples/json/optimized/json.go | 87 +- main.go | 6 +- pigeon.go | 222 ++- test/alternate_entrypoint/altentry.go | 110 +- test/andnot/andnot.go | 112 +- test/emptystate/emptystate.go | 114 +- test/errorpos/errorpos.go | 140 +- test/global_store/global_store.go | 112 +- test/goto/goto.go | 124 +- test/goto_state/goto_state.go | 124 +- test/issue_1/issue_1.go | 106 +- test/issue_18/issue_18.go | 110 +- test/issue_65/issue_65.go | 110 +- test/issue_65/optimized-grammar/issue_65.go | 104 +- test/issue_65/optimized/issue_65.go | 57 +- test/issue_70/issue_70.go | 108 +- test/issue_70/optimized-grammar/issue_70.go | 104 +- test/issue_70/optimized/issue_70.go | 55 +- test/issue_70b/issue_70b.go | 158 +- test/issue_80/issue_80.go | 106 +- test/labeled_failures/labeled_failures.go | 116 +- test/left_recursion/left_recursion.go | 1756 +++++++++++++++++ test/left_recursion/left_recursion.peg | 32 + test/left_recursion/left_recursion_test.go | 17 + test/linear/linear.go | 112 +- test/max_expr_cnt/maxexpr.go | 182 +- test/max_expr_cnt/maxexpr.peg | 9 +- test/predicates/predicates.go | 108 +- test/runeerror/runeerror.go | 104 +- test/state/state.go | 104 +- test/stateclone/stateclone.go | 116 +- test/statereadonly/statereadonly.go | 116 +- test/staterestore/optimized/staterestore.go | 55 +- test/staterestore/standard/staterestore.go | 120 +- test/staterestore/staterestore.go | 120 +- test/thrownrecover/thrownrecover.go | 150 +- 47 files changed, 5282 insertions(+), 1364 deletions(-) create mode 100644 test/left_recursion/left_recursion.go create mode 100644 test/left_recursion/left_recursion.peg create mode 100644 test/left_recursion/left_recursion_test.go diff --git a/Makefile b/Makefile index 0b33214b..d2c04fad 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ $(TEST_DIR)/goto_state/goto_state.go: $(TEST_DIR)/goto_state/goto_state.peg $(BI $(BINDIR)/pigeon -nolint $< > $@ $(TEST_DIR)/max_expr_cnt/maxexpr.go: $(TEST_DIR)/max_expr_cnt/maxexpr.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -ignore-left-recursion $< > $@ + $(BINDIR)/pigeon -nolint $< > $@ $(TEST_DIR)/labeled_failures/labeled_failures.go: $(TEST_DIR)/labeled_failures/labeled_failures.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ @@ -167,11 +167,14 @@ $(TEST_DIR)/issue_70/optimized-grammar/issue_70.go: $(TEST_DIR)/issue_70/issue_7 $(BINDIR)/pigeon -nolint -optimize-grammar $< > $@ $(TEST_DIR)/issue_70b/issue_70b.go: $(TEST_DIR)/issue_70b/issue_70b.peg $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint --optimize-grammar -ignore-left-recursion $< > $@ + $(BINDIR)/pigeon -nolint -optimize-grammar -support-left-recursion $< > $@ $(TEST_DIR)/issue_80/issue_80.go: $(TEST_DIR)/issue_80/issue_80.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ +$(TEST_DIR)/left_recursion/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ + lint: golint ./... go vet ./... diff --git a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go index fdfa0d70..98ad7d79 100644 --- a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go +++ b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go @@ -77,6 +77,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Initializer", @@ -102,6 +104,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Rule", @@ -167,6 +171,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Expression", @@ -175,6 +181,8 @@ var g = &grammar{ pos: position{line: 41, col: 14, offset: 958}, name: "ChoiceExpr", }, + leader: false, + leftRecursive: false, }, { name: "ChoiceExpr", @@ -226,6 +234,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ActionExpr", @@ -267,6 +277,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SeqExpr", @@ -308,6 +320,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LabeledExpr", @@ -360,6 +374,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrefixedExpr", @@ -402,6 +418,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrefixedOp", @@ -427,6 +445,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SuffixedExpr", @@ -469,6 +489,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SuffixedOp", @@ -500,6 +522,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrimaryExpr", @@ -566,6 +590,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RuleRefExpr", @@ -619,6 +645,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SemanticPredExpr", @@ -652,6 +680,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SemanticPredOp", @@ -677,6 +707,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RuleDefOp", @@ -710,6 +742,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SourceChar", @@ -717,6 +751,8 @@ var g = &grammar{ expr: &anyMatcher{ line: 160, col: 14, offset: 4172, }, + leader: false, + leftRecursive: false, }, { name: "Comment", @@ -734,6 +770,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MultiLineComment", @@ -776,6 +814,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MultiLineCommentNoLineTerminator", @@ -827,6 +867,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleLineComment", @@ -861,6 +903,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Identifier", @@ -869,6 +913,8 @@ var g = &grammar{ pos: position{line: 166, col: 14, offset: 4419}, name: "IdentifierName", }, + leader: false, + leftRecursive: false, }, { name: "IdentifierName", @@ -893,6 +939,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "IdentifierStart", @@ -905,6 +953,8 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "IdentifierPart", @@ -925,6 +975,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LitMatcher", @@ -959,6 +1011,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "StringLiteral", @@ -1038,6 +1092,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DoubleStringChar", @@ -1095,6 +1151,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleStringChar", @@ -1152,6 +1210,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RawStringChar", @@ -1174,6 +1234,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DoubleStringEscape", @@ -1193,6 +1255,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleStringEscape", @@ -1212,6 +1276,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CommonEscapeSequence", @@ -1241,6 +1307,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharEscape", @@ -1298,6 +1366,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OctalEscape", @@ -1319,6 +1389,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "HexEscape", @@ -1342,6 +1414,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LongUnicodeEscape", @@ -1389,6 +1463,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ShortUnicodeEscape", @@ -1420,6 +1496,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OctalDigit", @@ -1431,6 +1509,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "DecimalDigit", @@ -1442,6 +1522,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "HexDigit", @@ -1453,6 +1535,8 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "CharClassMatcher", @@ -1518,6 +1602,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ClassCharRange", @@ -1541,6 +1627,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ClassChar", @@ -1598,6 +1686,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CharClassEscape", @@ -1617,6 +1707,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "UnicodeClassEscape", @@ -1662,6 +1754,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharUnicodeClass", @@ -1673,6 +1767,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "UnicodeClass", @@ -1688,6 +1784,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "AnyMatcher", @@ -1702,6 +1800,8 @@ var g = &grammar{ want: "\".\"", }, }, + leader: false, + leftRecursive: false, }, { name: "CodeBlock", @@ -1731,6 +1831,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Code", @@ -1786,6 +1888,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "__", @@ -1810,6 +1914,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -1830,6 +1936,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Whitespace", @@ -1841,6 +1949,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "EOL", @@ -1851,6 +1961,8 @@ var g = &grammar{ ignoreCase: false, want: "\"\\n\"", }, + leader: false, + leftRecursive: false, }, { name: "EOS", @@ -1908,6 +2020,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -1918,6 +2032,8 @@ var g = &grammar{ line: 237, col: 8, offset: 6792, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -2473,6 +2589,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } type choiceExpr struct { @@ -2800,14 +2919,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -3008,7 +3132,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -3049,36 +3173,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -3090,6 +3230,15 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -3137,9 +3286,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -3149,7 +3295,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -3163,7 +3309,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -3192,7 +3338,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -3309,7 +3455,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -3327,7 +3473,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -3383,7 +3529,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -3401,7 +3547,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -3420,7 +3566,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -3440,7 +3586,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -3453,7 +3599,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -3483,7 +3629,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -3501,7 +3647,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -3516,7 +3662,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/builder/builder.go b/builder/builder.go index 9d067327..939b97f8 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -77,14 +77,13 @@ func Optimize(optimize bool) Option { } } -// IgnoreLeftRecursion returns an option that specifies the ignoreLeftRecursion option -// If ignoreLeftRecursion is true, errors associated -// with the use of left recursion rules are ignored. -func IgnoreLeftRecursion(ignore bool) Option { +// SupportLeftRecursion returns an option that specifies the supportLeftRecursion option +// If supportLeftRecursion is true, LeftRecursion code is added to the resulting parser +func SupportLeftRecursion(support bool) Option { return func(b *builder) Option { prev := b.optimize - b.ignoreLeftRecursion = ignore - return IgnoreLeftRecursion(prev) + b.supportLeftRecursion = support + return SupportLeftRecursion(prev) } } @@ -129,7 +128,8 @@ type builder struct { basicLatinLookupTable bool globalState bool nolint bool - ignoreLeftRecursion bool + supportLeftRecursion bool + haveLeftRecursion bool ruleName string exprIndex int @@ -145,11 +145,15 @@ func (b *builder) setOptions(opts []Option) { } func (b *builder) buildParser(grammar *ast.Grammar) error { - if err := PrepareGramma(grammar); err != nil { - if !b.ignoreLeftRecursion { - return fmt.Errorf("uncorrect gramma: %w", err) - } + haveLeftRecursion, err := PrepareGrammar(grammar) + if err != nil { + return fmt.Errorf("uncorrect gramma: %w", err) } + if !b.supportLeftRecursion && haveLeftRecursion { + return fmt.Errorf("uncorrect gramma: %w", ErrHaveLeftRecirsion) + } + b.haveLeftRecursion = haveLeftRecursion + b.writeInit(grammar.Init) b.writeGrammar(grammar) for _, rule := range grammar.Rules { @@ -199,6 +203,8 @@ func (b *builder) writeRule(r *ast.Rule) { b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) b.writef("\texpr: ") b.writeExpr(r.Expr) + b.writelnf("\tleader: %t,", r.Leader) + b.writelnf("\tleftRecursive: %t,", r.LeftRecursive) b.writelnf("},") } @@ -774,11 +780,13 @@ func (b *builder) writeStaticCode() { Optimize bool BasicLatinLookupTable bool GlobalState bool + LeftRecursion bool Nolint bool }{ Optimize: b.optimize, BasicLatinLookupTable: b.basicLatinLookupTable, GlobalState: b.globalState, + LeftRecursion: b.haveLeftRecursion, Nolint: b.nolint, } t := template.Must(template.New("static_code").Parse(staticCode)) diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index f408ef6f..ce535790 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -251,6 +251,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} @@ -504,6 +507,8 @@ type parser struct { debug bool memoize bool + // {{ end }} ==template== + // ==template== {{ if or .LeftRecursion (not .Optimize) }} // memoization table for the packrat algorithm: // map[offset in source] map[expression or rule] {value, match} memo map[int]map[any]resultTuple @@ -603,14 +608,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } // {{ end }} ==template== @@ -756,7 +766,7 @@ func (p *parser) sliceFrom(start savepoint) []byte { return p.data[start.position.offset:p.pt.position.offset] } -// ==template== {{ if not .Optimize }} +// ==template== {{ if or .LeftRecursion (not .Optimize) }} func (p *parser) getMemoized(node any) (resultTuple, bool) { if len(p.memo) == 0 { return resultTuple{}, false @@ -829,7 +839,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -870,41 +880,142 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +// ==template== {{ if .LeftRecursion }} +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + // ==template== {{ if not .Optimize }} if p.debug { - defer p.out(p.in("parseRule " + rule.name)) + defer p.out(p.in("recursive " + rule.name)) } + // {{ end }} ==template== + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + for { + // ==template== {{ if or .GlobalState (not .Optimize) }} + lastState := p.cloneState() + // {{ end }} ==template== + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + // ==template== {{ if not .Optimize }} + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + } + // {{ end }} ==template== + if (!ok) || (endMark.offset <= lastResult.end.offset) { + // ==template== {{ if or .GlobalState (not .Optimize) }} + p.restoreState(lastState) + // {{ end }} ==template== + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +// {{ end }} ==template== + +// ==template== {{ if not .Optimize }} +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +// {{ end }} ==template== + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + // ==template== {{ if not .Optimize }} + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + // {{ end }} ==template== + var ( + val any + ok bool + // ==template== {{ if not .Optimize }} + startMark = p.pt + // {{ end }} ==template== + ) + + // ==template== {{ if and .LeftRecursion (not .Optimize) }} + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + // {{ else if not .Optimize }} if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) + } + // {{ else if .LeftRecursion }} + if rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) } + } else { + val, ok = p.parseRule(rule) } + // {{ else }} + val, ok = p.parseRule(rule) + // {{ end }} ==template== - start := p.pt + // ==template== {{ if not .Optimize }} + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } // {{ end }} ==template== + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== return val, ok } -// {{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { // ==template== {{ if not .Optimize }} var pt savepoint @@ -918,7 +1029,18 @@ func (p *parser) parseExpr(expr any) (any, bool) { } // {{ end }} ==template== + val, ok := p.parseExpr(expr) + + // ==template== {{ if not .Optimize }} + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + // {{ end }} ==template== + return val, ok +} +// {{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -968,11 +1090,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - // ==template== {{ if not .Optimize }} - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== return val, ok } @@ -984,7 +1101,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { // {{ end }} ==template== start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1003,7 +1120,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } // ==template== {{ if not .Optimize }} if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } // {{ end }} ==template== return val, ok @@ -1043,7 +1160,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { state := p.cloneState() // {{ end }} ==template== p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(state) @@ -1187,7 +1304,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { // ==template== {{ if not .Optimize }} @@ -1213,7 +1330,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1281,7 +1398,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { // {{ end }} ==template== p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() // ==template== {{ if or .GlobalState (not .Optimize) }} @@ -1303,7 +1420,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1325,7 +1442,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { // {{ end }} ==template== p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1347,7 +1464,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1364,7 +1481,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { state := p.cloneState() // {{ end }} ==template== for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(state) @@ -1405,7 +1522,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1425,7 +1542,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1442,7 +1559,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/builder/left_recursion.go b/builder/left_recursion.go index a93c8b26..630815cc 100644 --- a/builder/left_recursion.go +++ b/builder/left_recursion.go @@ -15,27 +15,18 @@ var ( ErrHaveLeftRecirsion = errors.New("have left recursion") ) -// PrepareGramma evaluates parameters associated with left recursion -func PrepareGramma(grammar *ast.Grammar) error { +// PrepareGrammar evaluates parameters associated with left recursion +func PrepareGrammar(grammar *ast.Grammar) (bool, error) { mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } ComputeNullables(mapRules) - if err := ComputeLeftRecursives(mapRules); err != nil { - return fmt.Errorf("error compute left recursive: %w", err) + haveLeftRecursion, err := ComputeLeftRecursives(mapRules) + if err != nil { + return false, fmt.Errorf("error compute left recursive: %w", err) } - rulesWithLeftRecursion := []string{} - for _, rule := range grammar.Rules { - if rule.LeftRecursive { - rulesWithLeftRecursion = append(rulesWithLeftRecursion, rule.Name.Val) - } - } - if len(rulesWithLeftRecursion) > 0 { - return fmt.Errorf("%w: %v", ErrHaveLeftRecirsion, rulesWithLeftRecursion) - } - - return nil + return haveLeftRecursion, nil } // ComputeNullables evaluates nullable nodes @@ -80,9 +71,10 @@ func findLeader( } // ComputeLeftRecursives evaluates left recursion -func ComputeLeftRecursives(rules map[string]*ast.Rule) error { +func ComputeLeftRecursives(rules map[string]*ast.Rule) (bool, error) { graph := MakeFirstGraph(rules) vertices := make([]string, 0, len(graph)) + haveLeftRecursion := false for k := range graph { vertices = append(vertices, k) } @@ -91,10 +83,11 @@ func ComputeLeftRecursives(rules map[string]*ast.Rule) error { if len(scc) > 1 { for name := range scc { rules[name].LeftRecursive = true + haveLeftRecursion = true } leader, err := findLeader(graph, scc) if err != nil { - return fmt.Errorf("error find leader %v: %w", scc, err) + return false, fmt.Errorf("error find leader %v: %w", scc, err) } rules[leader].Leader = true } else { @@ -106,10 +99,11 @@ func ComputeLeftRecursives(rules map[string]*ast.Rule) error { if _, ok := graph[name][name]; ok { rules[name].LeftRecursive = true rules[name].Leader = true + haveLeftRecursion = true } } } - return nil + return haveLeftRecursion, nil } // MakeFirstGraph compute the graph of left-invocations. diff --git a/builder/left_recursion_test.go b/builder/left_recursion_test.go index 64aad87b..0442054c 100644 --- a/builder/left_recursion_test.go +++ b/builder/left_recursion_test.go @@ -13,7 +13,7 @@ import ( func TestLeftRrecursive(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = expr NEWLINE expr = ('-' term / expr '+' term / term) term = NUMBER @@ -22,12 +22,13 @@ func TestLeftRrecursive(t *testing.T) { baz = NAME? ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) require.NoError(t, err) - err = builder.PrepareGramma(g) - require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) - mapRules := make(map[string]*ast.Rule, len(g.Rules)) - for _, rule := range g.Rules { + haveLeftRecursion, err := builder.PrepareGrammar(grammar) + require.NoError(t, err) + require.True(t, haveLeftRecursion) + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } assert.False(t, mapRules["start"].LeftRecursive) @@ -40,17 +41,18 @@ func TestLeftRrecursive(t *testing.T) { func TestNullable(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = sign NUMBER sign = ('-' / '+')? ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) require.NoError(t, err) - err = builder.PrepareGramma(g) + haveLeftRecursion, err := builder.PrepareGrammar(grammar) require.NoError(t, err) - mapRules := make(map[string]*ast.Rule, len(g.Rules)) - for _, rule := range g.Rules { + require.False(t, haveLeftRecursion) + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } assert.False(t, mapRules["start"].Nullable) @@ -59,17 +61,18 @@ func TestNullable(t *testing.T) { func TestAdvancedLeftRrecursive(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = NUMBER / sign start sign = '-'? ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) + require.NoError(t, err) + haveLeftRecursion, err := builder.PrepareGrammar(grammar) require.NoError(t, err) - err = builder.PrepareGramma(g) - require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) - mapRules := make(map[string]*ast.Rule, len(g.Rules)) - for _, rule := range g.Rules { + require.True(t, haveLeftRecursion) + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } assert.False(t, mapRules["start"].Nullable) @@ -80,18 +83,19 @@ func TestAdvancedLeftRrecursive(t *testing.T) { func TestMutuallyLeftRrecursive(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = foo 'E' foo = bar 'A' / 'B' bar = foo 'C' / 'D' ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) require.NoError(t, err) - err = builder.PrepareGramma(g) - require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) - mapRules := make(map[string]*ast.Rule, len(g.Rules)) - for _, rule := range g.Rules { + haveLeftRecursion, err := builder.PrepareGrammar(grammar) + require.NoError(t, err) + require.True(t, haveLeftRecursion) + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } assert.False(t, mapRules["start"].LeftRecursive) @@ -101,18 +105,19 @@ func TestMutuallyLeftRrecursive(t *testing.T) { func TestNastyMutuallyLeftRrecursive(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = target '=' target = maybe '+' / NAME maybe = maybe '-' / target ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) + require.NoError(t, err) + haveLeftRecursion, err := builder.PrepareGrammar(grammar) require.NoError(t, err) - err = builder.PrepareGramma(g) - require.ErrorIs(t, err, builder.ErrHaveLeftRecirsion) - mapRules := make(map[string]*ast.Rule, len(g.Rules)) - for _, rule := range g.Rules { + require.True(t, haveLeftRecursion) + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) + for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } assert.False(t, mapRules["start"].LeftRecursive) @@ -122,15 +127,15 @@ func TestNastyMutuallyLeftRrecursive(t *testing.T) { func TestLeftRecursionTooComplex(t *testing.T) { t.Parallel() - grammar := ` + text := ` start = foo foo = bar '+' / baz '+' / '+' bar = baz '-' / foo '-' / '-' baz = foo '*' / bar '*' / '*' ` p := bootstrap.NewParser() - g, err := p.Parse("", strings.NewReader(grammar)) + grammar, err := p.Parse("", strings.NewReader(text)) require.NoError(t, err) - err = builder.PrepareGramma(g) + _, err = builder.PrepareGrammar(grammar) require.ErrorIs(t, err, builder.ErrNoLeader) } diff --git a/builder/static_code.go b/builder/static_code.go index 3628ae12..18c47975 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -269,6 +269,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} @@ -522,6 +525,8 @@ type parser struct { debug bool memoize bool + // {{ end }} ==template== + // ==template== {{ if or .LeftRecursion (not .Optimize) }} // memoization table for the packrat algorithm: // map[offset in source] map[expression or rule] {value, match} memo map[int]map[any]resultTuple @@ -621,14 +626,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } // {{ end }} ==template== @@ -774,7 +784,7 @@ func (p *parser) sliceFrom(start savepoint) []byte { return p.data[start.position.offset:p.pt.position.offset] } -// ==template== {{ if not .Optimize }} +// ==template== {{ if or .LeftRecursion (not .Optimize) }} func (p *parser) getMemoized(node any) (resultTuple, bool) { if len(p.memo) == 0 { return resultTuple{}, false @@ -847,7 +857,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -888,41 +898,142 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +// ==template== {{ if .LeftRecursion }} +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + // ==template== {{ if not .Optimize }} if p.debug { - defer p.out(p.in("parseRule " + rule.name)) + defer p.out(p.in("recursive " + rule.name)) } + // {{ end }} ==template== + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + for { + // ==template== {{ if or .GlobalState (not .Optimize) }} + lastState := p.cloneState() + // {{ end }} ==template== + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + // ==template== {{ if not .Optimize }} + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + } + // {{ end }} ==template== + if (!ok) || (endMark.offset <= lastResult.end.offset) { + // ==template== {{ if or .GlobalState (not .Optimize) }} + p.restoreState(lastState) + // {{ end }} ==template== + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +// {{ end }} ==template== + +// ==template== {{ if not .Optimize }} +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +// {{ end }} ==template== + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + // ==template== {{ if not .Optimize }} + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + // {{ end }} ==template== + var ( + val any + ok bool + // ==template== {{ if not .Optimize }} + startMark = p.pt + // {{ end }} ==template== + ) + + // ==template== {{ if and .LeftRecursion (not .Optimize) }} + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + // {{ else if not .Optimize }} if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) + } + // {{ else if .LeftRecursion }} + if rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) } + } else { + val, ok = p.parseRule(rule) } + // {{ else }} + val, ok = p.parseRule(rule) + // {{ end }} ==template== - start := p.pt + // ==template== {{ if not .Optimize }} + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } // {{ end }} ==template== + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - // ==template== {{ if not .Optimize }} - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== return val, ok } -// {{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { // ==template== {{ if not .Optimize }} var pt savepoint @@ -936,7 +1047,18 @@ func (p *parser) parseExpr(expr any) (any, bool) { } // {{ end }} ==template== + val, ok := p.parseExpr(expr) + + // ==template== {{ if not .Optimize }} + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + // {{ end }} ==template== + return val, ok +} +// {{ if .Nolint }} nolint: gocyclo {{else}} ==template== {{ end }} +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -986,11 +1108,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - // ==template== {{ if not .Optimize }} - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - // {{ end }} ==template== return val, ok } @@ -1002,7 +1119,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { // {{ end }} ==template== start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1021,7 +1138,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } // ==template== {{ if not .Optimize }} if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } // {{ end }} ==template== return val, ok @@ -1061,7 +1178,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { state := p.cloneState() // {{ end }} ==template== p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(state) @@ -1205,7 +1322,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { // ==template== {{ if not .Optimize }} @@ -1231,7 +1348,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1299,7 +1416,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { // {{ end }} ==template== p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() // ==template== {{ if or .GlobalState (not .Optimize) }} @@ -1321,7 +1438,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1343,7 +1460,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { // {{ end }} ==template== p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1365,7 +1482,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1382,7 +1499,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { state := p.cloneState() // {{ end }} ==template== for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(state) @@ -1423,7 +1540,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1443,7 +1560,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1460,7 +1577,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { // {{ end }} ==template== p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/examples/calculator/calculator.go b/examples/calculator/calculator.go index 4f4000a9..80654872 100644 --- a/examples/calculator/calculator.go +++ b/examples/calculator/calculator.go @@ -99,6 +99,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Expr", @@ -156,6 +158,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Term", @@ -205,6 +209,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Factor", @@ -255,6 +261,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "AddOp", @@ -280,6 +288,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MulOp", @@ -305,6 +315,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Integer", @@ -337,6 +349,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -352,6 +366,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -362,6 +378,8 @@ var g = &grammar{ line: 101, col: 9, offset: 1916, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -691,6 +709,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1034,14 +1055,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1243,7 +1269,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1284,37 +1310,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1326,6 +1367,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1373,9 +1424,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1385,7 +1433,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1399,7 +1447,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1428,7 +1476,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1546,7 +1594,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1564,7 +1612,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1620,7 +1668,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1638,7 +1686,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1657,7 +1705,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1677,7 +1725,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1690,7 +1738,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1720,7 +1768,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1738,7 +1786,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1753,7 +1801,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/examples/indentation/indentation.go b/examples/indentation/indentation.go index 57a4d48f..481ccf13 100644 --- a/examples/indentation/indentation.go +++ b/examples/indentation/indentation.go @@ -62,6 +62,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Statements", @@ -81,6 +83,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Line", @@ -106,6 +110,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ReturnOp", @@ -141,6 +147,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Statement", @@ -231,6 +239,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Assignment", @@ -280,6 +290,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LogicalExpression", @@ -296,6 +308,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "AdditiveExpression", @@ -345,6 +359,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrimaryExpression", @@ -370,6 +386,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Integer", @@ -388,6 +406,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Identifier", @@ -418,6 +438,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "AddOp", @@ -443,6 +465,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -457,6 +481,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOL", @@ -513,6 +539,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Comment", @@ -538,6 +566,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -548,6 +578,8 @@ var g = &grammar{ line: 44, col: 8, offset: 1832, }, }, + leader: false, + leftRecursive: false, }, { name: "INDENTATION", @@ -574,6 +606,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "INDENT", @@ -582,6 +616,8 @@ var g = &grammar{ pos: position{line: 48, col: 10, offset: 1948}, run: (*parser).callonINDENT1, }, + leader: false, + leftRecursive: false, }, { name: "DEDENT", @@ -590,6 +626,8 @@ var g = &grammar{ pos: position{line: 50, col: 10, offset: 2035}, run: (*parser).callonDEDENT1, }, + leader: false, + leftRecursive: false, }, }, } @@ -1004,6 +1042,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1347,14 +1388,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1556,7 +1602,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1597,37 +1643,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1639,6 +1700,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1686,9 +1757,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1698,7 +1766,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1712,7 +1780,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1741,7 +1809,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1859,7 +1927,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1877,7 +1945,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1933,7 +2001,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1951,7 +2019,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1970,7 +2038,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1990,7 +2058,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -2003,7 +2071,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -2033,7 +2101,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -2051,7 +2119,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -2066,7 +2134,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/examples/json/json.go b/examples/json/json.go index 6e6b17b0..51ada837 100644 --- a/examples/json/json.go +++ b/examples/json/json.go @@ -59,6 +59,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Value", @@ -109,6 +111,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Object", @@ -212,6 +216,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Array", @@ -279,6 +285,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Number", @@ -333,6 +341,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Integer", @@ -364,6 +374,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Exponent", @@ -396,6 +408,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "String", @@ -459,6 +473,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EscapedChar", @@ -471,6 +487,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "EscapeSequence", @@ -488,6 +506,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharEscape", @@ -499,6 +519,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "UnicodeEscape", @@ -530,6 +552,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DecimalDigit", @@ -541,6 +565,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "NonZeroDecimalDigit", @@ -552,6 +578,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "HexDigit", @@ -563,6 +591,8 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "Bool", @@ -592,6 +622,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Null", @@ -606,6 +638,8 @@ var g = &grammar{ want: "\"null\"", }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -621,6 +655,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -631,6 +667,8 @@ var g = &grammar{ line: 89, col: 8, offset: 2077, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -986,6 +1024,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1329,14 +1370,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1538,7 +1584,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1579,37 +1625,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1621,6 +1682,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1668,9 +1739,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1680,7 +1748,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1694,7 +1762,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1723,7 +1791,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1841,7 +1909,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1859,7 +1927,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1915,7 +1983,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1933,7 +2001,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1952,7 +2020,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1972,7 +2040,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1985,7 +2053,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -2015,7 +2083,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -2033,7 +2101,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -2048,7 +2116,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/examples/json/optimized-grammar/json.go b/examples/json/optimized-grammar/json.go index 2dff3d4f..03f0dc47 100644 --- a/examples/json/optimized-grammar/json.go +++ b/examples/json/optimized-grammar/json.go @@ -67,6 +67,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Value", @@ -356,6 +358,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Object", @@ -707,6 +711,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Array", @@ -786,6 +792,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -1163,6 +1171,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1506,14 +1517,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1715,7 +1731,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1756,37 +1772,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1798,6 +1829,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1845,9 +1886,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1857,7 +1895,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1871,7 +1909,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1900,7 +1938,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -2018,7 +2056,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -2036,7 +2074,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -2092,7 +2130,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -2110,7 +2148,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -2129,7 +2167,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -2149,7 +2187,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -2162,7 +2200,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -2192,7 +2230,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -2210,7 +2248,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -2225,7 +2263,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/examples/json/optimized/json.go b/examples/json/optimized/json.go index 013a8c0e..a59bdab0 100644 --- a/examples/json/optimized/json.go +++ b/examples/json/optimized/json.go @@ -58,6 +58,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Value", @@ -108,6 +110,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Object", @@ -211,6 +215,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Array", @@ -278,6 +284,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Number", @@ -332,6 +340,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Integer", @@ -363,6 +373,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Exponent", @@ -396,6 +408,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "String", @@ -459,6 +473,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EscapedChar", @@ -472,6 +488,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "EscapeSequence", @@ -489,6 +507,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharEscape", @@ -501,6 +521,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "UnicodeEscape", @@ -532,6 +554,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DecimalDigit", @@ -544,6 +568,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "NonZeroDecimalDigit", @@ -556,6 +582,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "HexDigit", @@ -568,6 +596,8 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "Bool", @@ -597,6 +627,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Null", @@ -611,6 +643,8 @@ var g = &grammar{ want: "\"null\"", }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -627,6 +661,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -637,6 +673,8 @@ var g = &grammar{ line: 89, col: 8, offset: 2077, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -921,6 +959,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1361,7 +1402,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1402,18 +1443,34 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + val, ok = p.parseRule(rule) + + return val, ok +} + func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + // nolint: gocyclo func (p *parser) parseExpr(expr any) (any, bool) { - p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1464,7 +1521,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1491,7 +1548,7 @@ func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restore(pt) @@ -1589,7 +1646,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { _ = altI p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { return val, ok @@ -1600,7 +1657,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1640,7 +1697,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { pt := p.pt p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restore(pt) @@ -1653,7 +1710,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1669,7 +1726,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1685,7 +1742,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1693,7 +1750,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restore(pt) return nil, false @@ -1707,7 +1764,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1721,7 +1778,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1732,7 +1789,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/main.go b/main.go index 8f13bf77..c5580450 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,7 @@ func main() { outputFlag = fs.String("o", "", "output file, defaults to stdout") optimizeBasicLatinFlag = fs.Bool("optimize-basic-latin", false, "generate optimized parser for Unicode Basic Latin character sets") optimizeGrammar = fs.Bool("optimize-grammar", false, "optimize the given grammar (EXPERIMENTAL FEATURE)") - ignoreLeftRecursion = fs.Bool("ignore-left-recursion", false, "ignore errors related to left recursion") + supportLeftRecursion = fs.Bool("support-left-recursion", false, "add support left recursion") optimizeParserFlag = fs.Bool("optimize-parser", false, "generate optimized parser without Debug and Memoize options") recvrNmFlag = fs.String("receiver-name", "c", "receiver name for the generated methods") noBuildFlag = fs.Bool("x", false, "do not build, only parse") @@ -137,10 +137,10 @@ func main() { optimizeParser := builder.Optimize(*optimizeParserFlag) basicLatinOptimize := builder.BasicLatinLookupTable(*optimizeBasicLatinFlag) nolintOpt := builder.Nolint(*nolint) - leftRecursionIgnorer := builder.IgnoreLeftRecursion(*ignoreLeftRecursion) + leftRecursionSupporter := builder.SupportLeftRecursion(*supportLeftRecursion) if err := builder.BuildParser( outBuf, grammar, curNmOpt, optimizeParser, basicLatinOptimize, - nolintOpt, leftRecursionIgnorer); err != nil { + nolintOpt, leftRecursionSupporter); err != nil { fmt.Fprintln(os.Stderr, "build error: ", err) exit(5) } diff --git a/pigeon.go b/pigeon.go index b38c26da..dfe6d04b 100644 --- a/pigeon.go +++ b/pigeon.go @@ -81,6 +81,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Initializer", @@ -106,6 +108,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Rule", @@ -171,6 +175,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Expression", @@ -179,6 +185,8 @@ var g = &grammar{ pos: position{line: 41, col: 14, offset: 962}, name: "RecoveryExpr", }, + leader: false, + leftRecursive: false, }, { name: "RecoveryExpr", @@ -248,6 +256,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Labels", @@ -299,6 +309,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ChoiceExpr", @@ -350,6 +362,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ActionExpr", @@ -391,6 +405,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SeqExpr", @@ -432,6 +448,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LabeledExpr", @@ -488,6 +506,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrefixedExpr", @@ -530,6 +550,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrefixedOp", @@ -555,6 +577,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SuffixedExpr", @@ -597,6 +621,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SuffixedOp", @@ -628,6 +654,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "PrimaryExpr", @@ -694,6 +722,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RuleRefExpr", @@ -747,6 +777,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SemanticPredExpr", @@ -780,6 +812,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SemanticPredOp", @@ -811,6 +845,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RuleDefOp", @@ -844,6 +880,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SourceChar", @@ -851,6 +889,8 @@ var g = &grammar{ expr: &anyMatcher{ line: 193, col: 14, offset: 5208, }, + leader: false, + leftRecursive: false, }, { name: "Comment", @@ -868,6 +908,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MultiLineComment", @@ -910,6 +952,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MultiLineCommentNoLineTerminator", @@ -961,6 +1005,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleLineComment", @@ -1004,6 +1050,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Identifier", @@ -1020,6 +1068,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "IdentifierName", @@ -1044,6 +1094,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "IdentifierStart", @@ -1056,6 +1108,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "IdentifierPart", @@ -1076,6 +1130,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LitMatcher", @@ -1110,6 +1166,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "StringLiteral", @@ -1290,6 +1348,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DoubleStringChar", @@ -1347,6 +1407,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleStringChar", @@ -1404,6 +1466,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "RawStringChar", @@ -1426,6 +1490,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "DoubleStringEscape", @@ -1471,6 +1537,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleStringEscape", @@ -1516,6 +1584,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CommonEscapeSequence", @@ -1545,6 +1615,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharEscape", @@ -1602,6 +1674,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OctalEscape", @@ -1658,6 +1732,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "HexEscape", @@ -1718,6 +1794,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "LongUnicodeEscape", @@ -1806,6 +1884,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ShortUnicodeEscape", @@ -1878,6 +1958,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OctalDigit", @@ -1889,6 +1971,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "DecimalDigit", @@ -1900,6 +1984,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "HexDigit", @@ -1911,6 +1997,8 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "CharClassMatcher", @@ -2028,6 +2116,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ClassCharRange", @@ -2051,6 +2141,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ClassChar", @@ -2108,6 +2200,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CharClassEscape", @@ -2167,6 +2261,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "UnicodeClassEscape", @@ -2293,6 +2389,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "SingleCharUnicodeClass", @@ -2304,6 +2402,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "AnyMatcher", @@ -2318,6 +2418,8 @@ var g = &grammar{ want: "\".\"", }, }, + leader: false, + leftRecursive: false, }, { name: "ThrowExpr", @@ -2391,6 +2493,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CodeBlock", @@ -2448,6 +2552,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Code", @@ -2512,6 +2618,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "__", @@ -2536,6 +2644,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -2556,6 +2666,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Whitespace", @@ -2567,6 +2679,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "EOL", @@ -2577,6 +2691,8 @@ var g = &grammar{ ignoreCase: false, want: "\"\\n\"", }, + leader: false, + leftRecursive: false, }, { name: "EOS", @@ -2634,6 +2750,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -2644,6 +2762,8 @@ var g = &grammar{ line: 334, col: 8, offset: 10147, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -3442,6 +3562,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -3785,14 +3908,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -3994,7 +4122,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -4035,37 +4163,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -4077,6 +4220,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -4124,9 +4277,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -4136,7 +4286,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -4150,7 +4300,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -4179,7 +4329,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -4297,7 +4447,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -4315,7 +4465,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -4371,7 +4521,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -4389,7 +4539,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -4408,7 +4558,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -4428,7 +4578,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -4441,7 +4591,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -4471,7 +4621,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -4489,7 +4639,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -4504,7 +4654,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/alternate_entrypoint/altentry.go b/test/alternate_entrypoint/altentry.go index fbe986e0..510a5253 100644 --- a/test/alternate_entrypoint/altentry.go +++ b/test/alternate_entrypoint/altentry.go @@ -59,6 +59,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Entry2", @@ -100,6 +102,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Entry3", @@ -132,6 +136,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "C", @@ -149,6 +155,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -460,6 +468,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -803,14 +814,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1012,7 +1028,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1053,37 +1069,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1095,6 +1126,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1142,9 +1183,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1154,7 +1192,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1168,7 +1206,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1197,7 +1235,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1315,7 +1353,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1333,7 +1371,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1389,7 +1427,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1407,7 +1445,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1426,7 +1464,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1446,7 +1484,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1459,7 +1497,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1489,7 +1527,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1507,7 +1545,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1522,7 +1560,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/andnot/andnot.go b/test/andnot/andnot.go index b0da4b84..b8ac7501 100644 --- a/test/andnot/andnot.go +++ b/test/andnot/andnot.go @@ -52,6 +52,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "AB", @@ -88,6 +90,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "CD", @@ -115,6 +119,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -129,6 +135,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -139,6 +147,8 @@ var g = &grammar{ line: 20, col: 8, offset: 388, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -400,6 +410,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -743,14 +756,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -952,7 +970,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -993,37 +1011,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1035,6 +1068,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1082,9 +1125,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1094,7 +1134,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1108,7 +1148,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1137,7 +1177,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1255,7 +1295,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1273,7 +1313,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1329,7 +1369,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1347,7 +1387,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1366,7 +1406,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1386,7 +1426,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1399,7 +1439,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1429,7 +1469,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1447,7 +1487,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1462,7 +1502,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/emptystate/emptystate.go b/test/emptystate/emptystate.go index e4573aff..88623211 100644 --- a/test/emptystate/emptystate.go +++ b/test/emptystate/emptystate.go @@ -55,6 +55,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "a", @@ -69,6 +71,8 @@ var g = &grammar{ want: "\"a\"", }, }, + leader: false, + leftRecursive: false, }, { name: "b", @@ -88,6 +92,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "c", @@ -102,6 +108,8 @@ var g = &grammar{ want: "\"c\"", }, }, + leader: false, + leftRecursive: false, }, { name: "d", @@ -121,6 +129,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "e", @@ -135,6 +145,8 @@ var g = &grammar{ want: "\"e\"", }, }, + leader: false, + leftRecursive: false, }, }, } @@ -460,6 +472,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -803,14 +818,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1012,7 +1032,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1053,37 +1073,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1095,6 +1130,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1142,9 +1187,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1154,7 +1196,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1168,7 +1210,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1197,7 +1239,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1315,7 +1357,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1333,7 +1375,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1389,7 +1431,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1407,7 +1449,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1426,7 +1468,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1446,7 +1488,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1459,7 +1501,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1489,7 +1531,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1507,7 +1549,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1522,7 +1564,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/errorpos/errorpos.go b/test/errorpos/errorpos.go index 0cda4812..505463e0 100644 --- a/test/errorpos/errorpos.go +++ b/test/errorpos/errorpos.go @@ -84,6 +84,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case01", @@ -132,6 +134,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case02", @@ -161,6 +165,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case03", @@ -196,6 +202,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case04", @@ -243,6 +251,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case05", @@ -277,6 +287,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case06", @@ -312,6 +324,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case07", @@ -366,6 +380,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case08", @@ -397,6 +413,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case09", @@ -429,6 +447,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case10", @@ -486,6 +506,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case11", @@ -523,6 +545,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "increment", @@ -533,6 +557,8 @@ var g = &grammar{ ignoreCase: false, want: "\"inc\"", }, + leader: false, + leftRecursive: false, }, { name: "decrement", @@ -543,6 +569,8 @@ var g = &grammar{ ignoreCase: false, want: "\"dec\"", }, + leader: false, + leftRecursive: false, }, { name: "zero", @@ -553,6 +581,8 @@ var g = &grammar{ ignoreCase: false, want: "\"zero\"", }, + leader: false, + leftRecursive: false, }, { name: "oneOrMore", @@ -563,6 +593,8 @@ var g = &grammar{ ignoreCase: false, want: "\"oneOrMore\"", }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -577,6 +609,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "__", @@ -588,6 +622,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -598,6 +634,8 @@ var g = &grammar{ line: 25, col: 8, offset: 725, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -839,6 +877,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1182,14 +1223,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1391,7 +1437,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1432,37 +1478,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1474,6 +1535,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1521,9 +1592,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1533,7 +1601,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1547,7 +1615,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1576,7 +1644,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1694,7 +1762,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1712,7 +1780,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1768,7 +1836,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1786,7 +1854,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1805,7 +1873,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1825,7 +1893,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1838,7 +1906,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1868,7 +1936,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1886,7 +1954,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1901,7 +1969,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/global_store/global_store.go b/test/global_store/global_store.go index 8768cdee..6c2c1dce 100644 --- a/test/global_store/global_store.go +++ b/test/global_store/global_store.go @@ -59,6 +59,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "increment", @@ -78,6 +80,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "decrement", @@ -97,6 +101,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "zero", @@ -116,6 +122,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -126,6 +134,8 @@ var g = &grammar{ line: 10, col: 8, offset: 473, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -421,6 +431,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -764,14 +777,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -973,7 +991,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1014,37 +1032,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1056,6 +1089,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1103,9 +1146,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1115,7 +1155,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1129,7 +1169,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1158,7 +1198,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1276,7 +1316,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1294,7 +1334,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1350,7 +1390,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1368,7 +1408,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1387,7 +1427,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1407,7 +1447,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1420,7 +1460,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1450,7 +1490,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1468,7 +1508,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1483,7 +1523,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/goto/goto.go b/test/goto/goto.go index 94e07c11..df1ab0e2 100644 --- a/test/goto/goto.go +++ b/test/goto/goto.go @@ -68,6 +68,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Line", @@ -110,6 +112,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Instruction", @@ -151,6 +155,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Label", @@ -178,6 +184,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "labelIdentifier", @@ -208,6 +216,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Noop", @@ -222,6 +232,8 @@ var g = &grammar{ want: "\"noop\"", }, }, + leader: false, + leftRecursive: false, }, { name: "Jump", @@ -253,6 +265,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "nl", @@ -268,6 +282,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "__", @@ -283,6 +299,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -298,6 +316,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -308,6 +328,8 @@ var g = &grammar{ line: 66, col: 8, offset: 1339, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -638,6 +660,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -981,14 +1006,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1190,7 +1220,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1231,37 +1261,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1273,6 +1318,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1320,9 +1375,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1332,7 +1384,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1346,7 +1398,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1375,7 +1427,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1493,7 +1545,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1511,7 +1563,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1567,7 +1619,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1585,7 +1637,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1604,7 +1656,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1624,7 +1676,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1637,7 +1689,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1667,7 +1719,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1685,7 +1737,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1700,7 +1752,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/goto_state/goto_state.go b/test/goto_state/goto_state.go index 3797cb42..ddfe1f62 100644 --- a/test/goto_state/goto_state.go +++ b/test/goto_state/goto_state.go @@ -72,6 +72,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Line", @@ -114,6 +116,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Instruction", @@ -155,6 +159,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Label", @@ -182,6 +188,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "labelIdentifier", @@ -212,6 +220,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Noop", @@ -226,6 +236,8 @@ var g = &grammar{ want: "\"noop\"", }, }, + leader: false, + leftRecursive: false, }, { name: "Jump", @@ -261,6 +273,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "nl", @@ -276,6 +290,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "__", @@ -291,6 +307,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -306,6 +324,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -316,6 +336,8 @@ var g = &grammar{ line: 60, col: 8, offset: 1463, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -666,6 +688,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1009,14 +1034,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1218,7 +1248,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1259,37 +1289,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1301,6 +1346,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1348,9 +1403,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1360,7 +1412,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1374,7 +1426,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1403,7 +1455,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1521,7 +1573,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1539,7 +1591,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1595,7 +1647,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1613,7 +1665,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1632,7 +1684,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1652,7 +1704,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1665,7 +1717,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1695,7 +1747,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1713,7 +1765,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1728,7 +1780,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_1/issue_1.go b/test/issue_1/issue_1.go index 66f1b7a0..e46a5c50 100644 --- a/test/issue_1/issue_1.go +++ b/test/issue_1/issue_1.go @@ -61,6 +61,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ID", @@ -79,6 +81,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -340,6 +344,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -683,14 +690,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -892,7 +904,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -933,37 +945,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -975,6 +1002,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1022,9 +1059,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1034,7 +1068,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1048,7 +1082,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1077,7 +1111,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1195,7 +1229,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1213,7 +1247,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1269,7 +1303,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1287,7 +1321,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1306,7 +1340,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1326,7 +1360,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1339,7 +1373,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1369,7 +1403,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1387,7 +1421,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1402,7 +1436,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_18/issue_18.go b/test/issue_18/issue_18.go index ba7e9e7c..e6fa9e73 100644 --- a/test/issue_18/issue_18.go +++ b/test/issue_18/issue_18.go @@ -80,6 +80,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "X", @@ -91,6 +93,8 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -106,6 +110,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -116,6 +122,8 @@ var g = &grammar{ line: 38, col: 9, offset: 403, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -357,6 +365,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -700,14 +711,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -909,7 +925,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -950,37 +966,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -992,6 +1023,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1039,9 +1080,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1051,7 +1089,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1065,7 +1103,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1094,7 +1132,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1212,7 +1250,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1230,7 +1268,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1286,7 +1324,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1304,7 +1342,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1323,7 +1361,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1343,7 +1381,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1356,7 +1394,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1386,7 +1424,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1404,7 +1442,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1419,7 +1457,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_65/issue_65.go b/test/issue_65/issue_65.go index b41c24f8..b90a78d9 100644 --- a/test/issue_65/issue_65.go +++ b/test/issue_65/issue_65.go @@ -37,6 +37,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "List", @@ -68,6 +70,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "X", @@ -90,6 +94,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Y", @@ -104,6 +110,8 @@ var g = &grammar{ want: "\"Y\"", }, }, + leader: false, + leftRecursive: false, }, }, } @@ -356,6 +364,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -699,14 +710,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -908,7 +924,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -949,37 +965,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -991,6 +1022,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1038,9 +1079,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1050,7 +1088,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1064,7 +1102,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1093,7 +1131,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1211,7 +1249,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1229,7 +1267,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1285,7 +1323,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1303,7 +1341,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1322,7 +1360,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1342,7 +1380,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1355,7 +1393,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1385,7 +1423,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1403,7 +1441,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1418,7 +1456,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_65/optimized-grammar/issue_65.go b/test/issue_65/optimized-grammar/issue_65.go index 2cbacf1b..36d0cfd2 100644 --- a/test/issue_65/optimized-grammar/issue_65.go +++ b/test/issue_65/optimized-grammar/issue_65.go @@ -79,6 +79,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -342,6 +344,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -685,14 +690,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -894,7 +904,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -935,37 +945,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -977,6 +1002,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1024,9 +1059,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1036,7 +1068,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1050,7 +1082,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1079,7 +1111,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1197,7 +1229,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1215,7 +1247,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1271,7 +1303,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1289,7 +1321,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1308,7 +1340,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1328,7 +1360,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1341,7 +1373,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1371,7 +1403,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1389,7 +1421,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1404,7 +1436,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_65/optimized/issue_65.go b/test/issue_65/optimized/issue_65.go index 81f882b3..e994c21b 100644 --- a/test/issue_65/optimized/issue_65.go +++ b/test/issue_65/optimized/issue_65.go @@ -36,6 +36,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "List", @@ -67,6 +69,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "X", @@ -89,6 +93,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Y", @@ -103,6 +109,8 @@ var g = &grammar{ want: "\"Y\"", }, }, + leader: false, + leftRecursive: false, }, }, } @@ -284,6 +292,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -724,7 +735,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -765,18 +776,34 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + val, ok = p.parseRule(rule) + + return val, ok +} + func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + // nolint: gocyclo func (p *parser) parseExpr(expr any) (any, bool) { - p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -827,7 +854,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -854,7 +881,7 @@ func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restore(pt) @@ -952,7 +979,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { _ = altI p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { return val, ok @@ -963,7 +990,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1003,7 +1030,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { pt := p.pt p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restore(pt) @@ -1016,7 +1043,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1032,7 +1059,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1048,7 +1075,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1056,7 +1083,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restore(pt) return nil, false @@ -1070,7 +1097,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1084,7 +1111,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1095,7 +1122,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_70/issue_70.go b/test/issue_70/issue_70.go index 4e2b1c3d..dbc2c860 100644 --- a/test/issue_70/issue_70.go +++ b/test/issue_70/issue_70.go @@ -45,6 +45,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Y", @@ -53,6 +55,8 @@ var g = &grammar{ pos: position{line: 6, col: 5, offset: 58}, name: "Z", }, + leader: false, + leftRecursive: false, }, { name: "Z", @@ -70,6 +74,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -331,6 +337,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -674,14 +683,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -883,7 +897,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -924,37 +938,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -966,6 +995,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1013,9 +1052,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1025,7 +1061,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1039,7 +1075,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1068,7 +1104,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1186,7 +1222,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1204,7 +1240,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1260,7 +1296,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1278,7 +1314,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1297,7 +1333,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1317,7 +1353,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1330,7 +1366,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1360,7 +1396,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1378,7 +1414,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1393,7 +1429,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_70/optimized-grammar/issue_70.go b/test/issue_70/optimized-grammar/issue_70.go index 59ff1536..1e30f4a0 100644 --- a/test/issue_70/optimized-grammar/issue_70.go +++ b/test/issue_70/optimized-grammar/issue_70.go @@ -54,6 +54,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -315,6 +317,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -658,14 +663,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -867,7 +877,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -908,37 +918,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -950,6 +975,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -997,9 +1032,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1009,7 +1041,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1023,7 +1055,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1052,7 +1084,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1170,7 +1202,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1188,7 +1220,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1244,7 +1276,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1262,7 +1294,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1281,7 +1313,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1301,7 +1333,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1314,7 +1346,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1344,7 +1376,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1362,7 +1394,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1377,7 +1409,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_70/optimized/issue_70.go b/test/issue_70/optimized/issue_70.go index b7012fde..518ce122 100644 --- a/test/issue_70/optimized/issue_70.go +++ b/test/issue_70/optimized/issue_70.go @@ -44,6 +44,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Y", @@ -52,6 +54,8 @@ var g = &grammar{ pos: position{line: 6, col: 5, offset: 58}, name: "Z", }, + leader: false, + leftRecursive: false, }, { name: "Z", @@ -69,6 +73,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -259,6 +265,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -699,7 +708,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -740,18 +749,34 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + val, ok = p.parseRule(rule) + + return val, ok +} + func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + // nolint: gocyclo func (p *parser) parseExpr(expr any) (any, bool) { - p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -802,7 +827,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -829,7 +854,7 @@ func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restore(pt) @@ -927,7 +952,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { _ = altI p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { return val, ok @@ -938,7 +963,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -978,7 +1003,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { pt := p.pt p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restore(pt) @@ -991,7 +1016,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1007,7 +1032,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1023,7 +1048,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1031,7 +1056,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restore(pt) return nil, false @@ -1045,7 +1070,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1059,7 +1084,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1070,7 +1095,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index d6e5b693..3dafd022 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -32,6 +32,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "D", @@ -55,6 +57,8 @@ var g = &grammar{ }, }, }, + leader: true, + leftRecursive: true, }, }, } @@ -316,6 +320,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -659,14 +666,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -868,7 +880,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -909,37 +921,102 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + if p.debug { - defer p.out(p.in("parseRule " + rule.name)) + defer p.out(p.in("recursive " + rule.name)) } - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) } + if (!ok) || (endMark.offset <= lastResult.end.offset) { + p.restoreState(lastState) + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ } - start := p.pt + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -951,6 +1028,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -998,9 +1085,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1010,7 +1094,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1024,7 +1108,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1053,7 +1137,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1171,7 +1255,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1189,7 +1273,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1245,7 +1329,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1263,7 +1347,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1282,7 +1366,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1302,7 +1386,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1315,7 +1399,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1345,7 +1429,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1363,7 +1447,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1378,7 +1462,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/issue_80/issue_80.go b/test/issue_80/issue_80.go index 2843b0e1..322fbf39 100644 --- a/test/issue_80/issue_80.go +++ b/test/issue_80/issue_80.go @@ -73,6 +73,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -83,6 +85,8 @@ var g = &grammar{ line: 17, col: 8, offset: 276, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -350,6 +354,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -693,14 +700,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -902,7 +914,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -943,37 +955,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -985,6 +1012,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1032,9 +1069,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1044,7 +1078,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1058,7 +1092,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1087,7 +1121,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1205,7 +1239,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1223,7 +1257,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1279,7 +1313,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1297,7 +1331,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1316,7 +1350,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1336,7 +1370,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1349,7 +1383,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1379,7 +1413,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1397,7 +1431,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1412,7 +1446,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/labeled_failures/labeled_failures.go b/test/labeled_failures/labeled_failures.go index 2c2e6fa6..b7fa8425 100644 --- a/test/labeled_failures/labeled_failures.go +++ b/test/labeled_failures/labeled_failures.go @@ -80,6 +80,8 @@ var g = &grammar{ "errId", }, }, + leader: false, + leftRecursive: false, }, { name: "List", @@ -124,6 +126,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ID", @@ -160,6 +164,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Comma", @@ -188,6 +194,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Sp", @@ -202,6 +210,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrComma", @@ -239,6 +249,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrID", @@ -276,6 +288,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -579,6 +593,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -922,14 +939,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1131,7 +1153,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1172,37 +1194,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1214,6 +1251,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1261,9 +1308,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1273,7 +1317,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1287,7 +1331,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1316,7 +1360,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1434,7 +1478,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1452,7 +1496,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1508,7 +1552,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1526,7 +1570,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1545,7 +1589,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1565,7 +1609,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1578,7 +1622,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1608,7 +1652,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1626,7 +1670,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1641,7 +1685,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/left_recursion.go new file mode 100644 index 00000000..e287dbfa --- /dev/null +++ b/test/left_recursion/left_recursion.go @@ -0,0 +1,1756 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursion + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 4, col: 1, offset: 28}, + expr: &actionExpr{ + pos: position{line: 4, col: 9, offset: 36}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 4, col: 9, offset: 36}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 4, col: 9, offset: 36}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 4, col: 11, offset: 38}, + name: "sum", + }, + }, + ¬Expr{ + pos: position{line: 4, col: 15, offset: 42}, + expr: &anyMatcher{ + line: 4, col: 16, offset: 43, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "sum", + pos: position{line: 7, col: 1, offset: 67}, + expr: &choiceExpr{ + pos: position{line: 7, col: 7, offset: 73}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 7, col: 7, offset: 73}, + run: (*parser).callonsum2, + expr: &seqExpr{ + pos: position{line: 7, col: 7, offset: 73}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 7, col: 7, offset: 73}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 7, col: 9, offset: 75}, + name: "sum", + }, + }, + &labeledExpr{ + pos: position{line: 7, col: 13, offset: 79}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 7, col: 17, offset: 83}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 7, col: 17, offset: 83}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 7, col: 21, offset: 87}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 7, col: 26, offset: 92}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 7, col: 28, offset: 94}, + name: "term", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 12, col: 5, offset: 232}, + run: (*parser).callonsum12, + expr: &labeledExpr{ + pos: position{line: 12, col: 5, offset: 232}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 12, col: 7, offset: 234}, + name: "term", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 15, col: 1, offset: 261}, + expr: &choiceExpr{ + pos: position{line: 15, col: 8, offset: 268}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 15, col: 8, offset: 268}, + run: (*parser).callonterm2, + expr: &seqExpr{ + pos: position{line: 15, col: 8, offset: 268}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 15, col: 8, offset: 268}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 15, col: 10, offset: 270}, + name: "term", + }, + }, + &labeledExpr{ + pos: position{line: 15, col: 15, offset: 275}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 15, col: 19, offset: 279}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 15, col: 19, offset: 279}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 15, col: 23, offset: 283}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 15, col: 27, offset: 287}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 15, col: 32, offset: 292}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 15, col: 34, offset: 294}, + name: "factor", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 20, col: 5, offset: 434}, + run: (*parser).callonterm13, + expr: &labeledExpr{ + pos: position{line: 20, col: 5, offset: 434}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 20, col: 7, offset: 436}, + name: "factor", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "factor", + pos: position{line: 23, col: 1, offset: 465}, + expr: &choiceExpr{ + pos: position{line: 23, col: 10, offset: 474}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 23, col: 10, offset: 474}, + run: (*parser).callonfactor2, + expr: &seqExpr{ + pos: position{line: 23, col: 10, offset: 474}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 23, col: 10, offset: 474}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 23, col: 14, offset: 478}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 23, col: 14, offset: 478}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 23, col: 18, offset: 482}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 23, col: 23, offset: 487}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 23, col: 25, offset: 489}, + name: "factor", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 27, col: 5, offset: 599}, + run: (*parser).callonfactor10, + expr: &labeledExpr{ + pos: position{line: 27, col: 5, offset: 599}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 27, col: 7, offset: 601}, + name: "atom", + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "atom", + pos: position{line: 30, col: 1, offset: 628}, + expr: &actionExpr{ + pos: position{line: 30, col: 8, offset: 635}, + run: (*parser).callonatom1, + expr: &oneOrMoreExpr{ + pos: position{line: 30, col: 8, offset: 635}, + expr: &charClassMatcher{ + pos: position{line: 30, col: 8, offset: 635}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onstart1(a any) (any, error) { + return a, nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1(stack["a"]) +} + +func (c *current) onsum2(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonsum2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onsum2(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onsum12(a any) (any, error) { + return a, nil +} + +func (p *parser) callonsum12() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onsum12(stack["a"]) +} + +func (c *current) onterm2(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonterm2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm2(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onterm13(a any) (any, error) { + return a, nil +} + +func (p *parser) callonterm13() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm13(stack["a"]) +} + +func (c *current) onfactor2(op, a any) (any, error) { + strA := a.(string) + strOp := string(op.([]byte)) + return "(" + strOp + strA + ")", nil +} + +func (p *parser) callonfactor2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor2(stack["op"], stack["a"]) +} + +func (c *current) onfactor10(a any) (any, error) { + return a, nil +} + +func (p *parser) callonfactor10() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor10(stack["a"]) +} + +func (c *current) onatom1() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonatom1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onatom1() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + randestak []any + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + } + if (!ok) || (endMark.offset <= lastResult.end.offset) { + p.restoreState(lastState) + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.randestak = append(p.randestak, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.randestak = p.randestak[:len(p.randestak)-1] + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + p.randestak = append(p.randestak, expr) + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + p.randestak = p.randestak[:len(p.randestak)-1] + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg new file mode 100644 index 00000000..ed00e554 --- /dev/null +++ b/test/left_recursion/left_recursion.peg @@ -0,0 +1,32 @@ +{ + package leftrecursion +} +start = a:sum !. { + return a, nil +} +sum = a:sum op:('+'/'-') b:term { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} / a:term { + return a, nil +} +term = a:term op:('*'/'/'/'%') b:factor { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} / a:factor { + return a, nil +} +factor = op:('+'/'-') a:factor { + strA := a.(string) + strOp := string(op.([]byte)) + return "(" + strOp + strA + ")", nil +} / a:atom { + return a, nil +} +atom = [0-9]+ { + return string(c.text), nil +} diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/left_recursion_test.go new file mode 100644 index 00000000..cf9695ed --- /dev/null +++ b/test/left_recursion/left_recursion_test.go @@ -0,0 +1,17 @@ +package leftrecursion + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLeftRecursion(t *testing.T) { + t.Parallel() + data := "7+10/2*-4+5*3%6-8*6" + res, err := Parse("", []byte(data)) + require.NoError(t, err) + str, ok := res.(string) + require.True(t, ok) + require.Equal(t, str, "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))") +} diff --git a/test/linear/linear.go b/test/linear/linear.go index db6940e6..a7b84606 100644 --- a/test/linear/linear.go +++ b/test/linear/linear.go @@ -139,6 +139,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "L", @@ -153,6 +155,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "N", @@ -167,6 +171,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "S", @@ -181,6 +187,8 @@ var g = &grammar{ inverted: false, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -191,6 +199,8 @@ var g = &grammar{ line: 11, col: 8, offset: 187, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -432,6 +442,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -775,14 +788,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -984,7 +1002,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1025,37 +1043,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1067,6 +1100,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1114,9 +1157,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1126,7 +1166,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1140,7 +1180,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1169,7 +1209,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1287,7 +1327,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1305,7 +1345,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1361,7 +1401,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1379,7 +1419,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1398,7 +1438,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1418,7 +1458,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1431,7 +1471,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1461,7 +1501,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1479,7 +1519,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1494,7 +1534,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/max_expr_cnt/maxexpr.go b/test/max_expr_cnt/maxexpr.go index 5f337e4c..565e1450 100644 --- a/test/max_expr_cnt/maxexpr.go +++ b/test/max_expr_cnt/maxexpr.go @@ -20,12 +20,86 @@ import ( var g = &grammar{ rules: []*rule{ { - name: "infinite_rule", + name: "long_rule1", pos: position{line: 6, col: 1, offset: 53}, expr: &ruleRefExpr{ - pos: position{line: 6, col: 17, offset: 69}, - name: "infinite_rule", + pos: position{line: 6, col: 14, offset: 66}, + name: "long_rule2", }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule2", + pos: position{line: 7, col: 1, offset: 77}, + expr: &ruleRefExpr{ + pos: position{line: 7, col: 14, offset: 90}, + name: "long_rule3", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule3", + pos: position{line: 8, col: 1, offset: 101}, + expr: &ruleRefExpr{ + pos: position{line: 8, col: 14, offset: 114}, + name: "long_rule4", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule4", + pos: position{line: 9, col: 1, offset: 125}, + expr: &ruleRefExpr{ + pos: position{line: 9, col: 14, offset: 138}, + name: "long_rule5", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule5", + pos: position{line: 10, col: 1, offset: 149}, + expr: &ruleRefExpr{ + pos: position{line: 10, col: 14, offset: 162}, + name: "long_rule6", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule6", + pos: position{line: 11, col: 1, offset: 173}, + expr: &ruleRefExpr{ + pos: position{line: 11, col: 14, offset: 186}, + name: "long_rule7", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule7", + pos: position{line: 12, col: 1, offset: 197}, + expr: &ruleRefExpr{ + pos: position{line: 12, col: 14, offset: 210}, + name: "long_rule8", + }, + leader: false, + leftRecursive: false, + }, + { + name: "long_rule8", + pos: position{line: 13, col: 1, offset: 221}, + expr: &litMatcher{ + pos: position{line: 13, col: 14, offset: 234}, + val: " ", + ignoreCase: false, + want: "\" \"", + }, + leader: false, + leftRecursive: false, }, }, } @@ -267,6 +341,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -610,14 +687,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -819,7 +901,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -860,37 +942,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -902,6 +999,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -949,9 +1056,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -961,7 +1065,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -975,7 +1079,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1004,7 +1108,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1122,7 +1226,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1140,7 +1244,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1196,7 +1300,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1214,7 +1318,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1233,7 +1337,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1253,7 +1357,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1266,7 +1370,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1296,7 +1400,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1314,7 +1418,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1329,7 +1433,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/max_expr_cnt/maxexpr.peg b/test/max_expr_cnt/maxexpr.peg index 804d2505..4b4558aa 100644 --- a/test/max_expr_cnt/maxexpr.peg +++ b/test/max_expr_cnt/maxexpr.peg @@ -3,4 +3,11 @@ package maxexprcnt } // trigger an infinite parse -infinite_rule = infinite_rule \ No newline at end of file +long_rule1 = long_rule2 +long_rule2 = long_rule3 +long_rule3 = long_rule4 +long_rule4 = long_rule5 +long_rule5 = long_rule6 +long_rule6 = long_rule7 +long_rule7 = long_rule8 +long_rule8 = " " \ No newline at end of file diff --git a/test/predicates/predicates.go b/test/predicates/predicates.go index 02e18ccb..f08d0194 100644 --- a/test/predicates/predicates.go +++ b/test/predicates/predicates.go @@ -84,6 +84,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "B", @@ -137,6 +139,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "C", @@ -172,6 +176,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -486,6 +492,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -829,14 +838,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1038,7 +1052,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1079,37 +1093,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1121,6 +1150,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1168,9 +1207,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1180,7 +1216,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1194,7 +1230,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1223,7 +1259,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1341,7 +1377,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1359,7 +1395,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1415,7 +1451,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1433,7 +1469,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1452,7 +1488,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1472,7 +1508,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1485,7 +1521,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1515,7 +1551,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1533,7 +1569,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1548,7 +1584,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/runeerror/runeerror.go b/test/runeerror/runeerror.go index 90182613..f9866379 100644 --- a/test/runeerror/runeerror.go +++ b/test/runeerror/runeerror.go @@ -78,6 +78,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -331,6 +333,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -674,14 +679,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -883,7 +893,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -924,37 +934,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -966,6 +991,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1013,9 +1048,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1025,7 +1057,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1039,7 +1071,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1068,7 +1100,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1186,7 +1218,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1204,7 +1236,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1260,7 +1292,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1278,7 +1310,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1297,7 +1329,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1317,7 +1349,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1330,7 +1362,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1360,7 +1392,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1378,7 +1410,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1393,7 +1425,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/state/state.go b/test/state/state.go index 99e860ed..564e13c7 100644 --- a/test/state/state.go +++ b/test/state/state.go @@ -115,6 +115,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -414,6 +416,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -757,14 +762,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -966,7 +976,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1007,37 +1017,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1049,6 +1074,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1096,9 +1131,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1108,7 +1140,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1122,7 +1154,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1151,7 +1183,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1269,7 +1301,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1287,7 +1319,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1343,7 +1375,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1361,7 +1393,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1380,7 +1412,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1400,7 +1432,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1413,7 +1445,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1443,7 +1475,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1461,7 +1493,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1476,7 +1508,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/stateclone/stateclone.go b/test/stateclone/stateclone.go index 23310b29..6bc0c8f5 100644 --- a/test/stateclone/stateclone.go +++ b/test/stateclone/stateclone.go @@ -75,6 +75,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "x", @@ -100,6 +102,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "y", @@ -125,6 +129,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "z", @@ -144,6 +150,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "c", @@ -163,6 +171,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "bc", @@ -182,6 +192,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ws", @@ -203,6 +215,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -504,6 +518,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -847,14 +864,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1056,7 +1078,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1097,37 +1119,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1139,6 +1176,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1186,9 +1233,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1198,7 +1242,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1212,7 +1256,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1241,7 +1285,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1359,7 +1403,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1377,7 +1421,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1433,7 +1477,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1451,7 +1495,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1470,7 +1514,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1490,7 +1534,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1503,7 +1547,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1533,7 +1577,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1551,7 +1595,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1566,7 +1610,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/statereadonly/statereadonly.go b/test/statereadonly/statereadonly.go index 07074289..e20c2cb8 100644 --- a/test/statereadonly/statereadonly.go +++ b/test/statereadonly/statereadonly.go @@ -67,6 +67,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "x", @@ -92,6 +94,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "y", @@ -117,6 +121,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "z", @@ -131,6 +137,8 @@ var g = &grammar{ want: "\"abcf\"", }, }, + leader: false, + leftRecursive: false, }, { name: "c", @@ -150,6 +158,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "bc", @@ -169,6 +179,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ws", @@ -190,6 +202,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -485,6 +499,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -828,14 +845,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1037,7 +1059,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1078,37 +1100,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1120,6 +1157,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1167,9 +1214,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1179,7 +1223,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1193,7 +1237,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1222,7 +1266,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1340,7 +1384,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1358,7 +1402,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1414,7 +1458,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1432,7 +1476,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1451,7 +1495,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1471,7 +1515,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1484,7 +1528,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1514,7 +1558,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1532,7 +1576,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1547,7 +1591,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/staterestore/optimized/staterestore.go b/test/staterestore/optimized/staterestore.go index 71e26e70..98d7a2a7 100644 --- a/test/staterestore/optimized/staterestore.go +++ b/test/staterestore/optimized/staterestore.go @@ -356,6 +356,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestAnd", @@ -400,6 +402,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestNot", @@ -455,6 +459,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -860,6 +866,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1350,7 +1359,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1391,18 +1400,34 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + val, ok = p.parseRule(rule) + + return val, ok +} + func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + // nolint: gocyclo func (p *parser) parseExpr(expr any) (any, bool) { - p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1455,7 +1480,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1487,7 +1512,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1578,7 +1603,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { return val, ok @@ -1590,7 +1615,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1634,7 +1659,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1648,7 +1673,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1664,7 +1689,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1680,7 +1705,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1689,7 +1714,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1712,7 +1737,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1726,7 +1751,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1737,7 +1762,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/staterestore/standard/staterestore.go b/test/staterestore/standard/staterestore.go index b490e8be..d406a9fe 100644 --- a/test/staterestore/standard/staterestore.go +++ b/test/staterestore/standard/staterestore.go @@ -62,6 +62,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestAnd", @@ -93,6 +95,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestNot", @@ -126,6 +130,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Z_", @@ -145,6 +151,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Expr", @@ -181,6 +189,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOL", @@ -200,6 +210,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Comment", @@ -233,6 +245,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -253,6 +267,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -263,6 +279,8 @@ var g = &grammar{ line: 46, col: 9, offset: 784, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -584,6 +602,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -927,14 +948,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1136,7 +1162,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1177,37 +1203,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1219,6 +1260,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1266,9 +1317,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1278,7 +1326,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1292,7 +1340,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1321,7 +1369,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1439,7 +1487,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1457,7 +1505,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1513,7 +1561,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1531,7 +1579,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1550,7 +1598,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1570,7 +1618,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1583,7 +1631,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1613,7 +1661,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1631,7 +1679,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1646,7 +1694,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/staterestore/staterestore.go b/test/staterestore/staterestore.go index b490e8be..d406a9fe 100644 --- a/test/staterestore/staterestore.go +++ b/test/staterestore/staterestore.go @@ -62,6 +62,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestAnd", @@ -93,6 +95,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "TestNot", @@ -126,6 +130,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Z_", @@ -145,6 +151,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Expr", @@ -181,6 +189,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOL", @@ -200,6 +210,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "Comment", @@ -233,6 +245,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "_", @@ -253,6 +267,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "EOF", @@ -263,6 +279,8 @@ var g = &grammar{ line: 46, col: 9, offset: 784, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -584,6 +602,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -927,14 +948,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1136,7 +1162,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1177,37 +1203,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -1219,6 +1260,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -1266,9 +1317,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -1278,7 +1326,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -1292,7 +1340,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -1321,7 +1369,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -1439,7 +1487,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -1457,7 +1505,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -1513,7 +1561,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -1531,7 +1579,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -1550,7 +1598,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -1570,7 +1618,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -1583,7 +1631,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -1613,7 +1661,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -1631,7 +1679,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -1646,7 +1694,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true diff --git a/test/thrownrecover/thrownrecover.go b/test/thrownrecover/thrownrecover.go index eb593986..f9e7ae88 100644 --- a/test/thrownrecover/thrownrecover.go +++ b/test/thrownrecover/thrownrecover.go @@ -43,6 +43,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case01", @@ -70,6 +72,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "MultiLabelRecover", @@ -89,6 +93,8 @@ var g = &grammar{ "errOther", }, }, + leader: false, + leftRecursive: false, }, { name: "number", @@ -129,6 +135,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "digit", @@ -180,6 +188,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrNonNumber", @@ -218,6 +228,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case02", @@ -246,6 +258,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ThrowUndefLabel", @@ -254,6 +268,8 @@ var g = &grammar{ pos: position{line: 33, col: 19, offset: 744}, label: "undeflabel", }, + leader: false, + leftRecursive: false, }, { name: "case03", @@ -281,6 +297,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OuterRecover03", @@ -310,6 +328,8 @@ var g = &grammar{ "errOther", }, }, + leader: false, + leftRecursive: false, }, { name: "InnerRecover03", @@ -328,6 +348,8 @@ var g = &grammar{ "errAlphaLower", }, }, + leader: false, + leftRecursive: false, }, { name: "number03", @@ -368,6 +390,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "digit03", @@ -446,6 +470,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrAlphaInner03", @@ -484,6 +510,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrAlphaOuter03", @@ -522,6 +550,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrOtherOuter03", @@ -560,6 +590,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "case04", @@ -587,6 +619,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "OuterRecover04", @@ -616,6 +650,8 @@ var g = &grammar{ "errOther", }, }, + leader: false, + leftRecursive: false, }, { name: "InnerRecover04", @@ -634,6 +670,8 @@ var g = &grammar{ "errAlphaLower", }, }, + leader: false, + leftRecursive: false, }, { name: "number04", @@ -674,6 +712,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "digit04", @@ -752,6 +792,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrAlphaInner04", @@ -760,6 +802,8 @@ var g = &grammar{ pos: position{line: 83, col: 19, offset: 2363}, run: (*parser).callonErrAlphaInner041, }, + leader: false, + leftRecursive: false, }, { name: "ErrAlphaOuter04", @@ -798,6 +842,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, { name: "ErrOtherOuter04", @@ -836,6 +882,8 @@ var g = &grammar{ }, }, }, + leader: false, + leftRecursive: false, }, }, } @@ -1366,6 +1414,9 @@ type rule struct { name string displayName string expr any + + leader bool + leftRecursive bool } // nolint: structcheck @@ -1709,14 +1760,19 @@ func (p *parser) print(prefix, s string) string { return s } +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + func (p *parser) in(s string) string { + res := p.printIndent(">", s) p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) + return res } func (p *parser) out(s string) string { p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) + return p.printIndent("<", s) } func (p *parser) addErr(err error) { @@ -1918,7 +1974,7 @@ func (p *parser) parse(g *grammar) (val any, err error) { } p.read() // advance to first rune - val, ok = p.parseRule(startRule) + val, ok = p.parseRuleWrap(startRule) if !ok { if len(*p.errs) == 0 { // If parsing fails, but no errors have been recorded, the expected values @@ -1959,37 +2015,52 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) parseRule(rule *rule) (any, bool) { +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { if p.debug { defer p.out(p.in("parseRule " + rule.name)) } + var ( + val any + ok bool + startMark = p.pt + ) if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRule(rule) } - start := p.pt + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { p.rstack = append(p.rstack, rule) p.pushV() - val, ok := p.parseExpr(rule.expr) + val, ok := p.parseExprWrap(rule.expr) p.popV() p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } return val, ok } -// nolint: gocyclo -func (p *parser) parseExpr(expr any) (any, bool) { +func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint if p.memoize { @@ -2001,6 +2072,16 @@ func (p *parser) parseExpr(expr any) (any, bool) { pt = p.pt } + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { p.ExprCnt++ if p.ExprCnt > p.maxExprCnt { panic(errMaxExprCnt) @@ -2048,9 +2129,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } return val, ok } @@ -2060,7 +2138,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { } start := p.pt - val, ok := p.parseExpr(act.expr) + val, ok := p.parseExprWrap(act.expr) if ok { p.cur.pos = start.position p.cur.text = p.sliceFrom(start) @@ -2074,7 +2152,7 @@ func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { val = actVal } if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + p.printIndent("MATCH", string(p.sliceFrom(start))) } return val, ok } @@ -2103,7 +2181,7 @@ func (p *parser) parseAndExpr(and *andExpr) (any, bool) { pt := p.pt state := p.cloneState() p.pushV() - _, ok := p.parseExpr(and.expr) + _, ok := p.parseExprWrap(and.expr) p.popV() p.restoreState(state) p.restore(pt) @@ -2221,7 +2299,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { state := p.cloneState() p.pushV() - val, ok := p.parseExpr(alt) + val, ok := p.parseExprWrap(alt) p.popV() if ok { p.incChoiceAltCnt(ch, altI) @@ -2239,7 +2317,7 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { } p.pushV() - val, ok := p.parseExpr(lab.expr) + val, ok := p.parseExprWrap(lab.expr) p.popV() if ok && lab.label != "" { m := p.vstack[len(p.vstack)-1] @@ -2295,7 +2373,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) { state := p.cloneState() p.pushV() p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) + _, ok := p.parseExprWrap(not.expr) p.maxFailInvertExpected = !p.maxFailInvertExpected p.popV() p.restoreState(state) @@ -2313,7 +2391,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { if len(vals) == 0 { @@ -2332,7 +2410,7 @@ func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { } p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) + val, ok := p.parseExprWrap(recover.expr) p.popRecovery() return val, ok @@ -2352,7 +2430,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) return nil, false } - return p.parseRule(rule) + return p.parseRuleWrap(rule) } func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { @@ -2365,7 +2443,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { pt := p.pt state := p.cloneState() for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) + val, ok := p.parseExprWrap(expr) if !ok { p.restoreState(state) p.restore(pt) @@ -2395,7 +2473,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { for i := len(p.recoveryStack) - 1; i >= 0; i-- { if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { return val, ok } } @@ -2413,7 +2491,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { for { p.pushV() - val, ok := p.parseExpr(expr.expr) + val, ok := p.parseExprWrap(expr.expr) p.popV() if !ok { return vals, true @@ -2428,7 +2506,7 @@ func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { } p.pushV() - val, _ := p.parseExpr(expr.expr) + val, _ := p.parseExprWrap(expr.expr) p.popV() // whether it matched or not, consider it a match return val, true From 96243990fd2fa37552a56d923211389efb21412c Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Fri, 16 Jun 2023 10:15:31 +0300 Subject: [PATCH 03/14] Add support priority of operations --- .../cmd/bootstrap-pigeon/bootstrap_pigeon.go | 41 +- builder/generated_static_code.go | 111 ++++- builder/static_code.go | 111 ++++- examples/calculator/calculator.go | 41 +- examples/indentation/indentation.go | 41 +- examples/json/json.go | 41 +- examples/json/optimized-grammar/json.go | 41 +- examples/json/optimized/json.go | 40 +- pigeon.go | 41 +- targeted_test.go | 2 +- test/alternate_entrypoint/altentry.go | 41 +- test/andnot/andnot.go | 41 +- test/emptystate/emptystate.go | 41 +- test/errorpos/errorpos.go | 41 +- test/global_store/global_store.go | 41 +- test/goto/goto.go | 41 +- test/goto_state/goto_state.go | 41 +- test/issue_1/issue_1.go | 41 +- test/issue_18/issue_18.go | 41 +- test/issue_65/issue_65.go | 41 +- test/issue_65/optimized-grammar/issue_65.go | 41 +- test/issue_65/optimized/issue_65.go | 40 +- test/issue_70/issue_70.go | 41 +- test/issue_70/optimized-grammar/issue_70.go | 41 +- test/issue_70/optimized/issue_70.go | 40 +- test/issue_70b/issue_70b.go | 108 ++++- test/issue_80/issue_80.go | 41 +- test/labeled_failures/labeled_failures.go | 41 +- test/left_recursion/left_recursion.go | 390 +++++++++--------- test/left_recursion/left_recursion.peg | 31 +- test/linear/linear.go | 41 +- test/max_expr_cnt/maxexpr.go | 41 +- test/predicates/predicates.go | 41 +- test/runeerror/runeerror.go | 41 +- test/state/state.go | 41 +- test/stateclone/stateclone.go | 41 +- test/statereadonly/statereadonly.go | 41 +- test/staterestore/optimized/staterestore.go | 40 +- test/staterestore/standard/staterestore.go | 41 +- test/staterestore/staterestore.go | 41 +- test/thrownrecover/thrownrecover.go | 41 +- 41 files changed, 1773 insertions(+), 411 deletions(-) diff --git a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go index 98ad7d79..e3da3d3d 100644 --- a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go +++ b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go @@ -2809,6 +2809,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + type parser struct { filename string pt savepoint @@ -2831,7 +2836,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -2884,6 +2889,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -2951,7 +2980,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -3210,11 +3239,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -3244,6 +3273,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -3286,6 +3316,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -3429,7 +3460,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index ce535790..e7ecf4af 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -492,6 +492,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // {{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} type parser struct { filename string @@ -519,7 +524,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -572,6 +577,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -642,7 +671,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -881,9 +910,74 @@ func listJoin(list []string, sep string, lastSep string) string { } // ==template== {{ if .LeftRecursion }} + +func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack); i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if indexPrev < 0 { + continue + } + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { + checkPriority, haveChoices := p.checkPrevChoice(rule) + if haveChoices && !checkPriority { + return nil, false + } p.restore(result.end) return result.v, result.b } @@ -910,7 +1004,8 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { // ==template== {{ if not .Optimize }} if p.debug { p.printIndent("RECURSIVE", fmt.Sprintf( - "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) } // {{ end }} ==template== if (!ok) || (endMark.offset <= lastResult.end.offset) { @@ -1007,11 +1102,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1046,6 +1141,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1090,6 +1186,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1272,7 +1369,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) @@ -1293,8 +1390,8 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { if p.debug { defer p.out(p.in("parseChoiceExpr")) } - // {{ end }} ==template== + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/builder/static_code.go b/builder/static_code.go index 18c47975..77d220cb 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -510,6 +510,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // {{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} type parser struct { filename string @@ -537,7 +542,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -590,6 +595,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -660,7 +689,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -899,9 +928,74 @@ func listJoin(list []string, sep string, lastSep string) string { } // ==template== {{ if .LeftRecursion }} + +func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack); i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if indexPrev < 0 { + continue + } + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { + checkPriority, haveChoices := p.checkPrevChoice(rule) + if haveChoices && !checkPriority { + return nil, false + } p.restore(result.end) return result.v, result.b } @@ -928,7 +1022,8 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { // ==template== {{ if not .Optimize }} if p.debug { p.printIndent("RECURSIVE", fmt.Sprintf( - "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) } // {{ end }} ==template== if (!ok) || (endMark.offset <= lastResult.end.offset) { @@ -1025,11 +1120,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1064,6 +1159,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1108,6 +1204,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1290,7 +1387,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) @@ -1311,8 +1408,8 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { if p.debug { defer p.out(p.in("parseChoiceExpr")) } - // {{ end }} ==template== + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/examples/calculator/calculator.go b/examples/calculator/calculator.go index 80654872..d1a1a53c 100644 --- a/examples/calculator/calculator.go +++ b/examples/calculator/calculator.go @@ -944,6 +944,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -967,7 +972,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1020,6 +1025,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1087,7 +1116,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1347,11 +1376,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1382,6 +1411,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1424,6 +1454,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1568,7 +1599,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/indentation/indentation.go b/examples/indentation/indentation.go index 481ccf13..d4bf0a1c 100644 --- a/examples/indentation/indentation.go +++ b/examples/indentation/indentation.go @@ -1277,6 +1277,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1300,7 +1305,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1353,6 +1358,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1420,7 +1449,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1680,11 +1709,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1715,6 +1744,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1757,6 +1787,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1901,7 +1932,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/json.go b/examples/json/json.go index 51ada837..62d49635 100644 --- a/examples/json/json.go +++ b/examples/json/json.go @@ -1259,6 +1259,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1282,7 +1287,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1335,6 +1340,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1402,7 +1431,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1662,11 +1691,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1697,6 +1726,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1739,6 +1769,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1883,7 +1914,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized-grammar/json.go b/examples/json/optimized-grammar/json.go index 03f0dc47..04962df7 100644 --- a/examples/json/optimized-grammar/json.go +++ b/examples/json/optimized-grammar/json.go @@ -1406,6 +1406,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1429,7 +1434,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1482,6 +1487,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1549,7 +1578,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1809,11 +1838,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1844,6 +1873,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1886,6 +1916,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -2030,7 +2061,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized/json.go b/examples/json/optimized/json.go index a59bdab0..aee781ff 100644 --- a/examples/json/optimized/json.go +++ b/examples/json/optimized/json.go @@ -1187,6 +1187,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1204,7 +1209,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1257,6 +1262,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1299,7 +1328,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1455,11 +1484,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1476,6 +1505,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1516,6 +1546,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1641,6 +1672,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/pigeon.go b/pigeon.go index dfe6d04b..98072648 100644 --- a/pigeon.go +++ b/pigeon.go @@ -3797,6 +3797,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -3820,7 +3825,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -3873,6 +3878,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -3940,7 +3969,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -4200,11 +4229,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -4235,6 +4264,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -4277,6 +4307,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -4421,7 +4452,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/targeted_test.go b/targeted_test.go index de2ee9bf..932f8ff2 100644 --- a/targeted_test.go +++ b/targeted_test.go @@ -766,7 +766,7 @@ func TestParseChoiceExpr(t *testing.T) { // add dummy rule to rule stack of parser r := rule{name: "dummy"} - p.rstack = append(p.rstack, &r) + p.pushRule(&r) // advance to the first rune p.read() diff --git a/test/alternate_entrypoint/altentry.go b/test/alternate_entrypoint/altentry.go index 510a5253..895b8ec3 100644 --- a/test/alternate_entrypoint/altentry.go +++ b/test/alternate_entrypoint/altentry.go @@ -703,6 +703,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -726,7 +731,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -779,6 +784,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -846,7 +875,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1106,11 +1135,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1141,6 +1170,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1183,6 +1213,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1327,7 +1358,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/andnot/andnot.go b/test/andnot/andnot.go index b8ac7501..32ff1d2b 100644 --- a/test/andnot/andnot.go +++ b/test/andnot/andnot.go @@ -645,6 +645,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -668,7 +673,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -721,6 +726,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -788,7 +817,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1048,11 +1077,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1083,6 +1112,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1125,6 +1155,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1269,7 +1300,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/emptystate/emptystate.go b/test/emptystate/emptystate.go index 88623211..aad4eeda 100644 --- a/test/emptystate/emptystate.go +++ b/test/emptystate/emptystate.go @@ -707,6 +707,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -730,7 +735,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -783,6 +788,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -850,7 +879,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1110,11 +1139,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1145,6 +1174,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1187,6 +1217,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1331,7 +1362,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/errorpos/errorpos.go b/test/errorpos/errorpos.go index 505463e0..a93fe8d0 100644 --- a/test/errorpos/errorpos.go +++ b/test/errorpos/errorpos.go @@ -1112,6 +1112,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1135,7 +1140,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1188,6 +1193,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1255,7 +1284,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1515,11 +1544,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1550,6 +1579,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1592,6 +1622,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1736,7 +1767,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/global_store/global_store.go b/test/global_store/global_store.go index 6c2c1dce..208c1644 100644 --- a/test/global_store/global_store.go +++ b/test/global_store/global_store.go @@ -666,6 +666,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -689,7 +694,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -742,6 +747,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -809,7 +838,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1069,11 +1098,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1104,6 +1133,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1146,6 +1176,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1290,7 +1321,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto/goto.go b/test/goto/goto.go index df1ab0e2..cf769282 100644 --- a/test/goto/goto.go +++ b/test/goto/goto.go @@ -895,6 +895,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -918,7 +923,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -971,6 +976,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1038,7 +1067,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1298,11 +1327,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1333,6 +1362,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1375,6 +1405,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1519,7 +1550,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto_state/goto_state.go b/test/goto_state/goto_state.go index ddfe1f62..1fb6ee0f 100644 --- a/test/goto_state/goto_state.go +++ b/test/goto_state/goto_state.go @@ -923,6 +923,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -946,7 +951,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -999,6 +1004,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1066,7 +1095,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1326,11 +1355,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1361,6 +1390,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1403,6 +1433,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1547,7 +1578,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_1/issue_1.go b/test/issue_1/issue_1.go index e46a5c50..1b0d9779 100644 --- a/test/issue_1/issue_1.go +++ b/test/issue_1/issue_1.go @@ -579,6 +579,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -602,7 +607,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -655,6 +660,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -722,7 +751,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -982,11 +1011,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1017,6 +1046,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1059,6 +1089,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1203,7 +1234,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_18/issue_18.go b/test/issue_18/issue_18.go index e6fa9e73..98275fb8 100644 --- a/test/issue_18/issue_18.go +++ b/test/issue_18/issue_18.go @@ -600,6 +600,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -623,7 +628,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -676,6 +681,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -743,7 +772,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1003,11 +1032,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1038,6 +1067,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1080,6 +1110,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1224,7 +1255,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/issue_65.go b/test/issue_65/issue_65.go index b90a78d9..0ad4b311 100644 --- a/test/issue_65/issue_65.go +++ b/test/issue_65/issue_65.go @@ -599,6 +599,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -622,7 +627,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -675,6 +680,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -742,7 +771,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1002,11 +1031,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1037,6 +1066,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1079,6 +1109,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1223,7 +1254,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized-grammar/issue_65.go b/test/issue_65/optimized-grammar/issue_65.go index 36d0cfd2..ce8ba010 100644 --- a/test/issue_65/optimized-grammar/issue_65.go +++ b/test/issue_65/optimized-grammar/issue_65.go @@ -579,6 +579,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -602,7 +607,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -655,6 +660,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -722,7 +751,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -982,11 +1011,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1017,6 +1046,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1059,6 +1089,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1203,7 +1234,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized/issue_65.go b/test/issue_65/optimized/issue_65.go index e994c21b..015691f1 100644 --- a/test/issue_65/optimized/issue_65.go +++ b/test/issue_65/optimized/issue_65.go @@ -520,6 +520,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -537,7 +542,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -590,6 +595,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -632,7 +661,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -788,11 +817,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -809,6 +838,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -849,6 +879,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -974,6 +1005,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/test/issue_70/issue_70.go b/test/issue_70/issue_70.go index dbc2c860..5f841f09 100644 --- a/test/issue_70/issue_70.go +++ b/test/issue_70/issue_70.go @@ -572,6 +572,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -595,7 +600,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -648,6 +653,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -715,7 +744,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -975,11 +1004,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1010,6 +1039,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1052,6 +1082,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1196,7 +1227,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized-grammar/issue_70.go b/test/issue_70/optimized-grammar/issue_70.go index 1e30f4a0..b7a5527f 100644 --- a/test/issue_70/optimized-grammar/issue_70.go +++ b/test/issue_70/optimized-grammar/issue_70.go @@ -552,6 +552,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -575,7 +580,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -628,6 +633,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -695,7 +724,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -955,11 +984,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -990,6 +1019,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1032,6 +1062,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1176,7 +1207,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized/issue_70.go b/test/issue_70/optimized/issue_70.go index 518ce122..7e535458 100644 --- a/test/issue_70/optimized/issue_70.go +++ b/test/issue_70/optimized/issue_70.go @@ -493,6 +493,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -510,7 +515,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -563,6 +568,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -605,7 +634,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -761,11 +790,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -782,6 +811,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -822,6 +852,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -947,6 +978,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index 3dafd022..f3863b0b 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -555,6 +555,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -578,7 +583,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -631,6 +636,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -698,7 +727,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -921,9 +950,73 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack); i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if indexPrev < 0 { + continue + } + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { + checkPriority, haveChoices := p.checkPrevChoice(rule) + if haveChoices && !checkPriority { + return nil, false + } p.restore(result.end) return result.v, result.b } @@ -945,7 +1038,8 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { endMark := p.pt if p.debug { p.printIndent("RECURSIVE", fmt.Sprintf( - "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) } if (!ok) || (endMark.offset <= lastResult.end.offset) { p.restoreState(lastState) @@ -1008,11 +1102,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1043,6 +1137,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1085,6 +1180,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1229,7 +1325,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_80/issue_80.go b/test/issue_80/issue_80.go index 322fbf39..35ca8aec 100644 --- a/test/issue_80/issue_80.go +++ b/test/issue_80/issue_80.go @@ -589,6 +589,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -612,7 +617,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -665,6 +670,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -732,7 +761,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -992,11 +1021,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1027,6 +1056,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1069,6 +1099,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1213,7 +1244,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/labeled_failures/labeled_failures.go b/test/labeled_failures/labeled_failures.go index b7fa8425..aa7f898c 100644 --- a/test/labeled_failures/labeled_failures.go +++ b/test/labeled_failures/labeled_failures.go @@ -828,6 +828,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -851,7 +856,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -904,6 +909,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -971,7 +1000,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1231,11 +1260,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1266,6 +1295,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1308,6 +1338,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1452,7 +1483,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/left_recursion.go index e287dbfa..8c9aec4d 100644 --- a/test/left_recursion/left_recursion.go +++ b/test/left_recursion/left_recursion.go @@ -21,25 +21,25 @@ var g = &grammar{ rules: []*rule{ { name: "start", - pos: position{line: 4, col: 1, offset: 28}, + pos: position{line: 5, col: 1, offset: 29}, expr: &actionExpr{ - pos: position{line: 4, col: 9, offset: 36}, + pos: position{line: 5, col: 9, offset: 37}, run: (*parser).callonstart1, expr: &seqExpr{ - pos: position{line: 4, col: 9, offset: 36}, + pos: position{line: 5, col: 9, offset: 37}, exprs: []any{ &labeledExpr{ - pos: position{line: 4, col: 9, offset: 36}, + pos: position{line: 5, col: 9, offset: 37}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 4, col: 11, offset: 38}, - name: "sum", + pos: position{line: 5, col: 11, offset: 39}, + name: "expr", }, }, ¬Expr{ - pos: position{line: 4, col: 15, offset: 42}, + pos: position{line: 5, col: 16, offset: 44}, expr: &anyMatcher{ - line: 4, col: 16, offset: 43, + line: 5, col: 17, offset: 45, }, }, }, @@ -49,114 +49,68 @@ var g = &grammar{ leftRecursive: false, }, { - name: "sum", - pos: position{line: 7, col: 1, offset: 67}, + name: "expr", + pos: position{line: 8, col: 1, offset: 66}, expr: &choiceExpr{ - pos: position{line: 7, col: 7, offset: 73}, + pos: position{line: 8, col: 8, offset: 73}, alternatives: []any{ &actionExpr{ - pos: position{line: 7, col: 7, offset: 73}, - run: (*parser).callonsum2, + pos: position{line: 8, col: 8, offset: 73}, + run: (*parser).callonexpr2, expr: &seqExpr{ - pos: position{line: 7, col: 7, offset: 73}, + pos: position{line: 8, col: 8, offset: 73}, exprs: []any{ - &labeledExpr{ - pos: position{line: 7, col: 7, offset: 73}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 7, col: 9, offset: 75}, - name: "sum", - }, - }, - &labeledExpr{ - pos: position{line: 7, col: 13, offset: 79}, - label: "op", - expr: &choiceExpr{ - pos: position{line: 7, col: 17, offset: 83}, - alternatives: []any{ - &litMatcher{ - pos: position{line: 7, col: 17, offset: 83}, - val: "+", - ignoreCase: false, - want: "\"+\"", - }, - &litMatcher{ - pos: position{line: 7, col: 21, offset: 87}, - val: "-", - ignoreCase: false, - want: "\"-\"", - }, - }, - }, + &litMatcher{ + pos: position{line: 8, col: 8, offset: 73}, + val: "-", + ignoreCase: false, + want: "\"-\"", }, &labeledExpr{ - pos: position{line: 7, col: 26, offset: 92}, - label: "b", + pos: position{line: 8, col: 12, offset: 77}, + label: "a", expr: &ruleRefExpr{ - pos: position{line: 7, col: 28, offset: 94}, - name: "term", + pos: position{line: 8, col: 14, offset: 79}, + name: "expr", }, }, }, }, }, &actionExpr{ - pos: position{line: 12, col: 5, offset: 232}, - run: (*parser).callonsum12, - expr: &labeledExpr{ - pos: position{line: 12, col: 5, offset: 232}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 12, col: 7, offset: 234}, - name: "term", - }, - }, - }, - }, - }, - leader: true, - leftRecursive: true, - }, - { - name: "term", - pos: position{line: 15, col: 1, offset: 261}, - expr: &choiceExpr{ - pos: position{line: 15, col: 8, offset: 268}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 15, col: 8, offset: 268}, - run: (*parser).callonterm2, + pos: position{line: 11, col: 5, offset: 152}, + run: (*parser).callonexpr7, expr: &seqExpr{ - pos: position{line: 15, col: 8, offset: 268}, + pos: position{line: 11, col: 5, offset: 152}, exprs: []any{ &labeledExpr{ - pos: position{line: 15, col: 8, offset: 268}, + pos: position{line: 11, col: 5, offset: 152}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 15, col: 10, offset: 270}, - name: "term", + pos: position{line: 11, col: 7, offset: 154}, + name: "expr", }, }, &labeledExpr{ - pos: position{line: 15, col: 15, offset: 275}, + pos: position{line: 11, col: 12, offset: 159}, label: "op", expr: &choiceExpr{ - pos: position{line: 15, col: 19, offset: 279}, + pos: position{line: 11, col: 16, offset: 163}, alternatives: []any{ &litMatcher{ - pos: position{line: 15, col: 19, offset: 279}, + pos: position{line: 11, col: 16, offset: 163}, val: "*", ignoreCase: false, want: "\"*\"", }, &litMatcher{ - pos: position{line: 15, col: 23, offset: 283}, + pos: position{line: 11, col: 20, offset: 167}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 15, col: 27, offset: 287}, + pos: position{line: 11, col: 24, offset: 171}, val: "%", ignoreCase: false, want: "\"%\"", @@ -165,59 +119,44 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 15, col: 32, offset: 292}, + pos: position{line: 11, col: 29, offset: 176}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 15, col: 34, offset: 294}, - name: "factor", + pos: position{line: 11, col: 31, offset: 178}, + name: "expr", }, }, }, }, }, &actionExpr{ - pos: position{line: 20, col: 5, offset: 434}, - run: (*parser).callonterm13, - expr: &labeledExpr{ - pos: position{line: 20, col: 5, offset: 434}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 20, col: 7, offset: 436}, - name: "factor", - }, - }, - }, - }, - }, - leader: true, - leftRecursive: true, - }, - { - name: "factor", - pos: position{line: 23, col: 1, offset: 465}, - expr: &choiceExpr{ - pos: position{line: 23, col: 10, offset: 474}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 23, col: 10, offset: 474}, - run: (*parser).callonfactor2, + pos: position{line: 16, col: 5, offset: 317}, + run: (*parser).callonexpr18, expr: &seqExpr{ - pos: position{line: 23, col: 10, offset: 474}, + pos: position{line: 16, col: 5, offset: 317}, exprs: []any{ &labeledExpr{ - pos: position{line: 23, col: 10, offset: 474}, + pos: position{line: 16, col: 5, offset: 317}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 16, col: 7, offset: 319}, + name: "expr", + }, + }, + &labeledExpr{ + pos: position{line: 16, col: 12, offset: 324}, label: "op", expr: &choiceExpr{ - pos: position{line: 23, col: 14, offset: 478}, + pos: position{line: 16, col: 16, offset: 328}, alternatives: []any{ &litMatcher{ - pos: position{line: 23, col: 14, offset: 478}, + pos: position{line: 16, col: 16, offset: 328}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 23, col: 18, offset: 482}, + pos: position{line: 16, col: 20, offset: 332}, val: "-", ignoreCase: false, want: "\"-\"", @@ -226,48 +165,40 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 23, col: 23, offset: 487}, - label: "a", + pos: position{line: 16, col: 25, offset: 337}, + label: "b", expr: &ruleRefExpr{ - pos: position{line: 23, col: 25, offset: 489}, - name: "factor", + pos: position{line: 16, col: 27, offset: 339}, + name: "expr", }, }, }, }, }, &actionExpr{ - pos: position{line: 27, col: 5, offset: 599}, - run: (*parser).callonfactor10, - expr: &labeledExpr{ - pos: position{line: 27, col: 5, offset: 599}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 27, col: 7, offset: 601}, - name: "atom", - }, + pos: position{line: 21, col: 5, offset: 477}, + run: (*parser).callonexpr28, + expr: &ruleRefExpr{ + pos: position{line: 21, col: 5, offset: 477}, + name: "term", }, }, }, }, - leader: false, - leftRecursive: false, + leader: true, + leftRecursive: true, }, { - name: "atom", - pos: position{line: 30, col: 1, offset: 628}, - expr: &actionExpr{ - pos: position{line: 30, col: 8, offset: 635}, - run: (*parser).callonatom1, - expr: &oneOrMoreExpr{ - pos: position{line: 30, col: 8, offset: 635}, - expr: &charClassMatcher{ - pos: position{line: 30, col: 8, offset: 635}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, + name: "term", + pos: position{line: 25, col: 1, offset: 518}, + expr: &oneOrMoreExpr{ + pos: position{line: 25, col: 8, offset: 525}, + expr: &charClassMatcher{ + pos: position{line: 25, col: 8, offset: 525}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, }, }, leader: false, @@ -286,82 +217,51 @@ func (p *parser) callonstart1() (any, error) { return p.cur.onstart1(stack["a"]) } -func (c *current) onsum2(a, op, b any) (any, error) { +func (c *current) onexpr2(a any) (any, error) { strA := a.(string) - strB := b.(string) - strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil + return "(" + "-" + strA + ")", nil } -func (p *parser) callonsum2() (any, error) { +func (p *parser) callonexpr2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onsum2(stack["a"], stack["op"], stack["b"]) + return p.cur.onexpr2(stack["a"]) } -func (c *current) onsum12(a any) (any, error) { - return a, nil -} - -func (p *parser) callonsum12() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onsum12(stack["a"]) -} - -func (c *current) onterm2(a, op, b any) (any, error) { +func (c *current) onexpr7(a, op, b any) (any, error) { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil } -func (p *parser) callonterm2() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onterm2(stack["a"], stack["op"], stack["b"]) -} - -func (c *current) onterm13(a any) (any, error) { - return a, nil -} - -func (p *parser) callonterm13() (any, error) { +func (p *parser) callonexpr7() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onterm13(stack["a"]) + return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) } -func (c *current) onfactor2(op, a any) (any, error) { +func (c *current) onexpr18(a, op, b any) (any, error) { strA := a.(string) + strB := b.(string) strOp := string(op.([]byte)) - return "(" + strOp + strA + ")", nil -} - -func (p *parser) callonfactor2() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onfactor2(stack["op"], stack["a"]) -} - -func (c *current) onfactor10(a any) (any, error) { - return a, nil + return "(" + strA + strOp + strB + ")", nil } -func (p *parser) callonfactor10() (any, error) { +func (p *parser) callonexpr18() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onfactor10(stack["a"]) + return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) } -func (c *current) onatom1() (any, error) { +func (c *current) onexpr28() (any, error) { return string(c.text), nil } -func (p *parser) callonatom1() (any, error) { +func (p *parser) callonexpr28() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onatom1() + return p.cur.onexpr28() } var ( @@ -836,6 +736,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -859,9 +764,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule - - randestak []any + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -914,6 +817,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -981,7 +908,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1204,9 +1131,73 @@ func listJoin(list []string, sep string, lastSep string) string { } } +func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack); i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if indexPrev < 0 { + continue + } + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { + checkPriority, haveChoices := p.checkPrevChoice(rule) + if haveChoices && !checkPriority { + return nil, false + } p.restore(result.end) return result.v, result.b } @@ -1228,7 +1219,8 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { endMark := p.pt if p.debug { p.printIndent("RECURSIVE", fmt.Sprintf( - "Rule %s depth %d: %t to %d", rule.name, depth, ok, endMark.offset)) + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) } if (!ok) || (endMark.offset <= lastResult.end.offset) { p.restoreState(lastState) @@ -1291,13 +1283,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) - p.randestak = append(p.randestak, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.randestak = p.randestak[:len(p.randestak)-1] - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1328,7 +1318,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.randestak = append(p.randestak, expr) + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1371,7 +1361,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.randestak = p.randestak[:len(p.randestak)-1] + p.popExpr() return val, ok } @@ -1516,7 +1506,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg index ed00e554..a8c2d9a2 100644 --- a/test/left_recursion/left_recursion.peg +++ b/test/left_recursion/left_recursion.peg @@ -1,32 +1,25 @@ { package leftrecursion } -start = a:sum !. { - return a, nil + +start = a:expr !. { + return a, nil } -sum = a:sum op:('+'/'-') b:term { +expr = '-' a:expr { + strA := a.(string) + return "(" + "-" + strA + ")", nil +} / a:expr op:('*'/'/'/'%') b:expr { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil -} / a:term { - return a, nil -} -term = a:term op:('*'/'/'/'%') b:factor { + return "(" + strA + strOp + strB + ")", nil +} / a:expr op:('+'/'-') b:expr { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil -} / a:factor { - return a, nil -} -factor = op:('+'/'-') a:factor { - strA := a.(string) - strOp := string(op.([]byte)) - return "(" + strOp + strA + ")", nil -} / a:atom { - return a, nil -} -atom = [0-9]+ { +} / term { return string(c.text), nil } + +term = [0-9]+ \ No newline at end of file diff --git a/test/linear/linear.go b/test/linear/linear.go index a7b84606..0c8c8982 100644 --- a/test/linear/linear.go +++ b/test/linear/linear.go @@ -677,6 +677,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -700,7 +705,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -753,6 +758,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -820,7 +849,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1080,11 +1109,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1115,6 +1144,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1157,6 +1187,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1301,7 +1332,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/max_expr_cnt/maxexpr.go b/test/max_expr_cnt/maxexpr.go index 565e1450..37af9ae1 100644 --- a/test/max_expr_cnt/maxexpr.go +++ b/test/max_expr_cnt/maxexpr.go @@ -576,6 +576,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -599,7 +604,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -652,6 +657,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -719,7 +748,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -979,11 +1008,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1014,6 +1043,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1056,6 +1086,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1200,7 +1231,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/predicates/predicates.go b/test/predicates/predicates.go index f08d0194..6cf95c2f 100644 --- a/test/predicates/predicates.go +++ b/test/predicates/predicates.go @@ -727,6 +727,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -750,7 +755,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -803,6 +808,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -870,7 +899,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1130,11 +1159,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1165,6 +1194,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1207,6 +1237,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1351,7 +1382,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/runeerror/runeerror.go b/test/runeerror/runeerror.go index f9866379..c2fd891b 100644 --- a/test/runeerror/runeerror.go +++ b/test/runeerror/runeerror.go @@ -568,6 +568,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -591,7 +596,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -644,6 +649,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -711,7 +740,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -971,11 +1000,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1006,6 +1035,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1048,6 +1078,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1192,7 +1223,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/state/state.go b/test/state/state.go index 564e13c7..0223a5d1 100644 --- a/test/state/state.go +++ b/test/state/state.go @@ -651,6 +651,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -674,7 +679,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -727,6 +732,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -794,7 +823,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1054,11 +1083,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1089,6 +1118,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1131,6 +1161,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1275,7 +1306,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/stateclone/stateclone.go b/test/stateclone/stateclone.go index 6bc0c8f5..7c0c44b3 100644 --- a/test/stateclone/stateclone.go +++ b/test/stateclone/stateclone.go @@ -753,6 +753,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -776,7 +781,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -829,6 +834,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -896,7 +925,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1156,11 +1185,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1191,6 +1220,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1233,6 +1263,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1377,7 +1408,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/statereadonly/statereadonly.go b/test/statereadonly/statereadonly.go index e20c2cb8..c78e5858 100644 --- a/test/statereadonly/statereadonly.go +++ b/test/statereadonly/statereadonly.go @@ -734,6 +734,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -757,7 +762,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -810,6 +815,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -877,7 +906,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1137,11 +1166,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1172,6 +1201,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1214,6 +1244,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1358,7 +1389,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/optimized/staterestore.go b/test/staterestore/optimized/staterestore.go index 98d7a2a7..75bb0f2d 100644 --- a/test/staterestore/optimized/staterestore.go +++ b/test/staterestore/optimized/staterestore.go @@ -1101,6 +1101,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1118,7 +1123,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1171,6 +1176,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1213,7 +1242,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1412,11 +1441,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1433,6 +1462,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1475,6 +1505,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1596,6 +1627,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + for altI, alt := range ch.alternatives { // dummy assignment to prevent compile error if optimized _ = altI diff --git a/test/staterestore/standard/staterestore.go b/test/staterestore/standard/staterestore.go index d406a9fe..b07459b3 100644 --- a/test/staterestore/standard/staterestore.go +++ b/test/staterestore/standard/staterestore.go @@ -837,6 +837,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -860,7 +865,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -913,6 +918,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -980,7 +1009,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1240,11 +1269,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1275,6 +1304,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1317,6 +1347,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1461,7 +1492,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/staterestore.go b/test/staterestore/staterestore.go index d406a9fe..b07459b3 100644 --- a/test/staterestore/staterestore.go +++ b/test/staterestore/staterestore.go @@ -837,6 +837,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -860,7 +865,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -913,6 +918,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -980,7 +1009,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1240,11 +1269,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -1275,6 +1304,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1317,6 +1347,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -1461,7 +1492,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/thrownrecover/thrownrecover.go b/test/thrownrecover/thrownrecover.go index f9e7ae88..e3a96858 100644 --- a/test/thrownrecover/thrownrecover.go +++ b/test/thrownrecover/thrownrecover.go @@ -1649,6 +1649,11 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +type ruleWithExpsStack struct { + rule *rule + estack []any +} + // nolint: structcheck,maligned type parser struct { filename string @@ -1672,7 +1677,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []*rule + rstack []ruleWithExpsStack // parse fail maxFailPos position @@ -1725,6 +1730,30 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack)-1] +} + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1792,7 +1821,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1] + rule := p.rstack[len(p.rstack)-1].rule if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -2052,11 +2081,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.rstack = append(p.rstack, rule) + p.pushRule(rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] + p.popRule() return val, ok } @@ -2087,6 +2116,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -2129,6 +2159,7 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + p.popExpr() return val, ok } @@ -2273,7 +2304,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) From 23894e22445b6cd1261d353a73e3935bc8d50bac Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Sun, 25 Jun 2023 11:44:42 +0300 Subject: [PATCH 04/14] Fix comments --- ast/ast.go | 142 ++++++++-------- ast/ast_optimize.go | 6 +- .../cmd/bootstrap-pigeon/bootstrap_pigeon.go | 116 ------------- builder/builder.go | 10 +- builder/left_recursion.go | 6 +- builder/left_recursion_test.go | 158 +++++++++++++----- builder/scc.go | 2 +- builder/scc_test.go | 133 ++++++++++++++- examples/calculator/calculator.go | 18 -- examples/indentation/indentation.go | 38 ----- examples/json/json.go | 38 ----- examples/json/optimized-grammar/json.go | 8 - examples/json/optimized/json.go | 38 ----- go.mod | 6 - go.sum | 27 --- pigeon.go | 120 ------------- test/alternate_entrypoint/altentry.go | 8 - test/andnot/andnot.go | 10 -- test/emptystate/emptystate.go | 12 -- test/errorpos/errorpos.go | 38 ----- test/global_store/global_store.go | 10 -- test/goto/goto.go | 22 --- test/goto_state/goto_state.go | 22 --- test/issue_1/issue_1.go | 4 - test/issue_18/issue_18.go | 8 - test/issue_65/issue_65.go | 8 - test/issue_65/optimized-grammar/issue_65.go | 2 - test/issue_65/optimized/issue_65.go | 8 - test/issue_70/issue_70.go | 6 - test/issue_70/optimized-grammar/issue_70.go | 2 - test/issue_70/optimized/issue_70.go | 6 - test/issue_80/issue_80.go | 4 - test/labeled_failures/labeled_failures.go | 14 -- test/left_recursion/left_recursion.go | 38 ++--- test/left_recursion/left_recursion.peg | 6 +- test/left_recursion/left_recursion_test.go | 19 ++- test/linear/linear.go | 10 -- test/max_expr_cnt/maxexpr.go | 34 +--- test/max_expr_cnt/maxexpr.peg | 4 +- test/predicates/predicates.go | 6 - test/runeerror/runeerror.go | 2 - test/state/state.go | 2 - test/stateclone/stateclone.go | 14 -- test/statereadonly/statereadonly.go | 14 -- test/staterestore/optimized/staterestore.go | 6 - test/staterestore/standard/staterestore.go | 18 -- test/staterestore/staterestore.go | 18 -- test/thrownrecover/thrownrecover.go | 48 ------ 48 files changed, 366 insertions(+), 923 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 6302fcc0..19831898 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -58,17 +58,17 @@ func (g *Grammar) String() string { return buf.String() } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (g *Grammar) NullableVisit(rules map[string]*Rule) bool { panic("NullableVisit should not be called on the Grammar") } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (g *Grammar) IsNullable() bool { panic("IsNullable should not be called on the Grammar") } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (g *Grammar) InitialNames() map[string]bool { panic("InitialNames should not be called on the Grammar") } @@ -81,7 +81,7 @@ type Rule struct { DisplayName *StringLit Expr Expression - // for work with left recursion + // Fields below to work with left recursion. Visited bool Nullable bool LeftRecursive bool @@ -105,7 +105,7 @@ func (r *Rule) String() string { r.p, r, r.Name, r.DisplayName, r.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (r *Rule) NullableVisit(rules map[string]*Rule) bool { if r.Visited { // A left-recursive rule is considered non-nullable. @@ -116,12 +116,12 @@ func (r *Rule) NullableVisit(rules map[string]*Rule) bool { return r.Nullable } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (r *Rule) IsNullable() bool { return r.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (r *Rule) InitialNames() map[string]bool { return r.Expr.InitialNames() } @@ -168,7 +168,7 @@ func (c *ChoiceExpr) String() string { return buf.String() } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (c *ChoiceExpr) NullableVisit(rules map[string]*Rule) bool { for _, alt := range c.Alternatives { if alt.NullableVisit(rules) { @@ -180,12 +180,12 @@ func (c *ChoiceExpr) NullableVisit(rules map[string]*Rule) bool { return false } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (c *ChoiceExpr) IsNullable() bool { return c.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (c *ChoiceExpr) InitialNames() map[string]bool { names := make(map[string]bool) for _, alt := range c.Alternatives { @@ -196,7 +196,7 @@ func (c *ChoiceExpr) InitialNames() map[string]bool { return names } -// FailureLabel is an identifier, which can by thrown and recovered in a grammar +// FailureLabel is an identifier, which can by thrown and recovered in a grammar. type FailureLabel string // RecoveryExpr is an ordered sequence of expressions. The parser tries to @@ -234,18 +234,18 @@ func (r *RecoveryExpr) String() string { return buf.String() } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (r *RecoveryExpr) NullableVisit(rules map[string]*Rule) bool { r.Nullable = r.Expr.NullableVisit(rules) || r.RecoverExpr.NullableVisit(rules) return r.Nullable } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (r *RecoveryExpr) IsNullable() bool { return r.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (r *RecoveryExpr) InitialNames() map[string]bool { names := make(map[string]bool) for name := range r.Expr.InitialNames() { @@ -283,18 +283,18 @@ func (a *ActionExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v, Code: %v}", a.p, a, a.Expr, a.Code) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (a *ActionExpr) NullableVisit(rules map[string]*Rule) bool { a.Nullable = a.Expr.NullableVisit(rules) return a.Nullable } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (a *ActionExpr) IsNullable() bool { return a.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (a *ActionExpr) InitialNames() map[string]bool { names := make(map[string]bool) for name := range a.Expr.InitialNames() { @@ -325,17 +325,17 @@ func (t *ThrowExpr) String() string { return fmt.Sprintf("%s: %T{Label: %v}", t.p, t, t.Label) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (t *ThrowExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (t *ThrowExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (t *ThrowExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -371,7 +371,7 @@ func (s *SeqExpr) String() string { return buf.String() } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (s *SeqExpr) NullableVisit(rules map[string]*Rule) bool { for _, item := range s.Exprs { if !item.NullableVisit(rules) { @@ -383,12 +383,12 @@ func (s *SeqExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (s *SeqExpr) IsNullable() bool { return s.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (s *SeqExpr) InitialNames() map[string]bool { names := make(map[string]bool) for _, item := range s.Exprs { @@ -426,17 +426,17 @@ func (l *LabeledExpr) String() string { return fmt.Sprintf("%s: %T{Label: %v, Expr: %v}", l.p, l, l.Label, l.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (l *LabeledExpr) NullableVisit(rules map[string]*Rule) bool { return l.Expr.NullableVisit(rules) } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (l *LabeledExpr) IsNullable() bool { return l.Expr.IsNullable() } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (l *LabeledExpr) InitialNames() map[string]bool { return l.Expr.InitialNames() } @@ -463,17 +463,17 @@ func (a *AndExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", a.p, a, a.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (a *AndExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (a *AndExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (a *AndExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -500,17 +500,17 @@ func (n *NotExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", n.p, n, n.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (n *NotExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (n *NotExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (n *NotExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -537,17 +537,17 @@ func (z *ZeroOrOneExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (z *ZeroOrOneExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (z *ZeroOrOneExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (z *ZeroOrOneExpr) InitialNames() map[string]bool { return z.Expr.InitialNames() } @@ -574,17 +574,17 @@ func (z *ZeroOrMoreExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", z.p, z, z.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (z *ZeroOrMoreExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (z *ZeroOrMoreExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (z *ZeroOrMoreExpr) InitialNames() map[string]bool { return z.Expr.InitialNames() } @@ -611,17 +611,17 @@ func (o *OneOrMoreExpr) String() string { return fmt.Sprintf("%s: %T{Expr: %v}", o.p, o, o.Expr) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (o *OneOrMoreExpr) NullableVisit(rules map[string]*Rule) bool { return false } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (o *OneOrMoreExpr) IsNullable() bool { return false } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (o *OneOrMoreExpr) InitialNames() map[string]bool { return o.Expr.InitialNames() } @@ -650,7 +650,7 @@ func (r *RuleRefExpr) String() string { return fmt.Sprintf("%s: %T{Name: %v}", r.p, r, r.Name) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (r *RuleRefExpr) NullableVisit(rules map[string]*Rule) bool { item, ok := rules[r.Name.Val] if !ok { @@ -662,12 +662,12 @@ func (r *RuleRefExpr) NullableVisit(rules map[string]*Rule) bool { return r.Nullable } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (r *RuleRefExpr) IsNullable() bool { return r.Nullable } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (r *RuleRefExpr) InitialNames() map[string]bool { return map[string]bool{r.Name.Val: true} } @@ -695,17 +695,17 @@ func (s *StateCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", s.p, s, s.Code) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (s *StateCodeExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (s *StateCodeExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (s *StateCodeExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -734,17 +734,17 @@ func (a *AndCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", a.p, a, a.Code) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (a *AndCodeExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (a *AndCodeExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (a *AndCodeExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -773,17 +773,17 @@ func (n *NotCodeExpr) String() string { return fmt.Sprintf("%s: %T{Code: %v}", n.p, n, n.Code) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (n *NotCodeExpr) NullableVisit(rules map[string]*Rule) bool { return true } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (n *NotCodeExpr) IsNullable() bool { return true } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (n *NotCodeExpr) InitialNames() map[string]bool { return make(map[string]bool) } @@ -812,18 +812,18 @@ func (l *LitMatcher) String() string { return fmt.Sprintf("%s: %T{Val: %q, IgnoreCase: %t}", l.p, l, l.Val, l.IgnoreCase) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (l *LitMatcher) NullableVisit(rules map[string]*Rule) bool { return l.IsNullable() } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (l *LitMatcher) IsNullable() bool { // The string token '' is considered empty. return len(l.Val) == 0 } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (l *LitMatcher) InitialNames() map[string]bool { return make(map[string]bool) } @@ -965,17 +965,17 @@ func (c *CharClassMatcher) String() string { c.p, c, c.Val, c.IgnoreCase, c.Inverted) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (c *CharClassMatcher) NullableVisit(rules map[string]*Rule) bool { return c.IsNullable() } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (c *CharClassMatcher) IsNullable() bool { return len(c.Chars) == 0 && len(c.Ranges) == 0 && len(c.UnicodeClasses) == 0 } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (c *CharClassMatcher) InitialNames() map[string]bool { return make(map[string]bool) } @@ -1001,17 +1001,17 @@ func (a *AnyMatcher) String() string { return fmt.Sprintf("%s: %T{Val: %q}", a.p, a, a.Val) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (a *AnyMatcher) NullableVisit(rules map[string]*Rule) bool { return false } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (a *AnyMatcher) IsNullable() bool { return false } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (a *AnyMatcher) InitialNames() map[string]bool { return make(map[string]bool) } @@ -1037,17 +1037,17 @@ func (c *CodeBlock) String() string { return fmt.Sprintf("%s: %T{Val: %q}", c.p, c, c.Val) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (c *CodeBlock) NullableVisit(rules map[string]*Rule) bool { panic("NullableVisit should not be called on the CodeBlock") } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (c *CodeBlock) IsNullable() bool { panic("IsNullable should not be called on the CodeBlock") } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (c *CodeBlock) InitialNames() map[string]bool { panic("InitialNames should not be called on the CodeBlock") } @@ -1073,17 +1073,17 @@ func (i *Identifier) String() string { return fmt.Sprintf("%s: %T{Val: %q}", i.p, i, i.Val) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (i *Identifier) NullableVisit(rules map[string]*Rule) bool { panic("NullableVisit should not be called on the Identifier") } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (i *Identifier) IsNullable() bool { panic("IsNullable should not be called on the Identifier") } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (i *Identifier) InitialNames() map[string]bool { panic("InitialNames should not be called on the Identifier") } @@ -1109,17 +1109,17 @@ func (s *StringLit) String() string { return fmt.Sprintf("%s: %T{Val: %q}", s.p, s, s.Val) } -// NullableVisit recursively determines whether an object is nullable +// NullableVisit recursively determines whether an object is nullable. func (s *StringLit) NullableVisit(rules map[string]*Rule) bool { panic("NullableVisit should not be called on the StringLit") } -// IsNullable returns the nullable attribute of the node +// IsNullable returns the nullable attribute of the node. func (s *StringLit) IsNullable() bool { panic("IsNullable should not be called on the StringLit") } -// InitialNames returns names of nodes with which an expression can begin +// InitialNames returns names of nodes with which an expression can begin. func (s *StringLit) InitialNames() map[string]bool { panic("InitialNames should not be called on the StringLit") } diff --git a/ast/ast_optimize.go b/ast/ast_optimize.go index 4687c8bd..0949adb5 100644 --- a/ast/ast_optimize.go +++ b/ast/ast_optimize.go @@ -34,7 +34,7 @@ func newGrammarOptimizer(protectedRules []string) *grammarOptimizer { // Visit is a generic Visitor to be used with Walk // The actual function, which should be used during Walk -// is held in ruleRefOptimizer.visitor +// is held in ruleRefOptimizer.visitor. func (r *grammarOptimizer) Visit(expr Expression) Visitor { return r.visitor(expr) } @@ -265,7 +265,7 @@ func (r *grammarOptimizer) optimizeRule(expr Expression) Expression { // cloneExpr takes an Expression and deep clones it (including all children) // This is necessary because referenced Rules are denormalized and therefore -// have to become independent from their original Expression +// have to become independent from their original Expression. func cloneExpr(expr Expression) Expression { switch expr := expr.(type) { case *ActionExpr: @@ -359,7 +359,7 @@ func cloneExpr(expr Expression) Expression { // The purpose of this function is to cleanup the redundancies created by the // optimize Visitor. This includes to remove redundant entries in Chars, Ranges // and UnicodeClasses of the given CharClassMatcher as well as regenerating the -// correct content for the Val field (string representation of the CharClassMatcher) +// correct content for the Val field (string representation of the CharClassMatcher). func (r *grammarOptimizer) cleanupCharClassMatcher(expr0 Expression) Visitor { // We are only interested in nodes of type *CharClassMatcher if chr, ok := expr0.(*CharClassMatcher); ok { diff --git a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go index e3da3d3d..5ee1176e 100644 --- a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go +++ b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go @@ -77,8 +77,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Initializer", @@ -104,8 +102,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Rule", @@ -171,8 +167,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Expression", @@ -181,8 +175,6 @@ var g = &grammar{ pos: position{line: 41, col: 14, offset: 958}, name: "ChoiceExpr", }, - leader: false, - leftRecursive: false, }, { name: "ChoiceExpr", @@ -234,8 +226,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ActionExpr", @@ -277,8 +267,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SeqExpr", @@ -320,8 +308,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LabeledExpr", @@ -374,8 +360,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrefixedExpr", @@ -418,8 +402,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrefixedOp", @@ -445,8 +427,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SuffixedExpr", @@ -489,8 +469,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SuffixedOp", @@ -522,8 +500,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrimaryExpr", @@ -590,8 +566,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RuleRefExpr", @@ -645,8 +619,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SemanticPredExpr", @@ -680,8 +652,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SemanticPredOp", @@ -707,8 +677,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RuleDefOp", @@ -742,8 +710,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SourceChar", @@ -751,8 +717,6 @@ var g = &grammar{ expr: &anyMatcher{ line: 160, col: 14, offset: 4172, }, - leader: false, - leftRecursive: false, }, { name: "Comment", @@ -770,8 +734,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MultiLineComment", @@ -814,8 +776,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MultiLineCommentNoLineTerminator", @@ -867,8 +827,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleLineComment", @@ -903,8 +861,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Identifier", @@ -913,8 +869,6 @@ var g = &grammar{ pos: position{line: 166, col: 14, offset: 4419}, name: "IdentifierName", }, - leader: false, - leftRecursive: false, }, { name: "IdentifierName", @@ -939,8 +893,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "IdentifierStart", @@ -953,8 +905,6 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "IdentifierPart", @@ -975,8 +925,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LitMatcher", @@ -1011,8 +959,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "StringLiteral", @@ -1092,8 +1038,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DoubleStringChar", @@ -1151,8 +1095,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleStringChar", @@ -1210,8 +1152,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RawStringChar", @@ -1234,8 +1174,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DoubleStringEscape", @@ -1255,8 +1193,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleStringEscape", @@ -1276,8 +1212,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CommonEscapeSequence", @@ -1307,8 +1241,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharEscape", @@ -1366,8 +1298,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OctalEscape", @@ -1389,8 +1319,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "HexEscape", @@ -1414,8 +1342,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LongUnicodeEscape", @@ -1463,8 +1389,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ShortUnicodeEscape", @@ -1496,8 +1420,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OctalDigit", @@ -1509,8 +1431,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "DecimalDigit", @@ -1522,8 +1442,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "HexDigit", @@ -1535,8 +1453,6 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "CharClassMatcher", @@ -1602,8 +1518,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ClassCharRange", @@ -1627,8 +1541,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ClassChar", @@ -1686,8 +1598,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CharClassEscape", @@ -1707,8 +1617,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "UnicodeClassEscape", @@ -1754,8 +1662,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharUnicodeClass", @@ -1767,8 +1673,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "UnicodeClass", @@ -1784,8 +1688,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "AnyMatcher", @@ -1800,8 +1702,6 @@ var g = &grammar{ want: "\".\"", }, }, - leader: false, - leftRecursive: false, }, { name: "CodeBlock", @@ -1831,8 +1731,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Code", @@ -1888,8 +1786,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "__", @@ -1914,8 +1810,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -1936,8 +1830,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Whitespace", @@ -1949,8 +1841,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "EOL", @@ -1961,8 +1851,6 @@ var g = &grammar{ ignoreCase: false, want: "\"\\n\"", }, - leader: false, - leftRecursive: false, }, { name: "EOS", @@ -2020,8 +1908,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -2032,8 +1918,6 @@ var g = &grammar{ line: 237, col: 8, offset: 6792, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/builder/builder.go b/builder/builder.go index 939b97f8..f93d2489 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -77,8 +77,8 @@ func Optimize(optimize bool) Option { } } -// SupportLeftRecursion returns an option that specifies the supportLeftRecursion option -// If supportLeftRecursion is true, LeftRecursion code is added to the resulting parser +// SupportLeftRecursion returns an option that specifies the supportLeftRecursion option. +// If supportLeftRecursion is true, LeftRecursion code is added to the resulting parser. func SupportLeftRecursion(support bool) Option { return func(b *builder) Option { prev := b.optimize @@ -203,8 +203,10 @@ func (b *builder) writeRule(r *ast.Rule) { b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) b.writef("\texpr: ") b.writeExpr(r.Expr) - b.writelnf("\tleader: %t,", r.Leader) - b.writelnf("\tleftRecursive: %t,", r.LeftRecursive) + if b.supportLeftRecursion { + b.writelnf("\tleader: %t,", r.Leader) + b.writelnf("\tleftRecursive: %t,", r.LeftRecursive) + } b.writelnf("},") } diff --git a/builder/left_recursion.go b/builder/left_recursion.go index 630815cc..6d668656 100644 --- a/builder/left_recursion.go +++ b/builder/left_recursion.go @@ -15,7 +15,7 @@ var ( ErrHaveLeftRecirsion = errors.New("have left recursion") ) -// PrepareGrammar evaluates parameters associated with left recursion +// PrepareGrammar evaluates parameters associated with left recursion. func PrepareGrammar(grammar *ast.Grammar) (bool, error) { mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { @@ -29,7 +29,7 @@ func PrepareGrammar(grammar *ast.Grammar) (bool, error) { return haveLeftRecursion, nil } -// ComputeNullables evaluates nullable nodes +// ComputeNullables evaluates nullable nodes. func ComputeNullables(rules map[string]*ast.Rule) { // Compute which rules in a grammar are nullable for _, rule := range rules { @@ -70,7 +70,7 @@ func findLeader( return leader, nil } -// ComputeLeftRecursives evaluates left recursion +// ComputeLeftRecursives evaluates left recursion. func ComputeLeftRecursives(rules map[string]*ast.Rule) (bool, error) { graph := MakeFirstGraph(rules) vertices := make([]string, 0, len(graph)) diff --git a/builder/left_recursion_test.go b/builder/left_recursion_test.go index 0442054c..99ac1bc7 100644 --- a/builder/left_recursion_test.go +++ b/builder/left_recursion_test.go @@ -1,18 +1,18 @@ package builder_test import ( + "errors" "strings" "testing" "github.com/mna/pigeon/ast" "github.com/mna/pigeon/bootstrap" "github.com/mna/pigeon/builder" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestLeftRrecursive(t *testing.T) { +func TestLeftRecursive(t *testing.T) { t.Parallel() + text := ` start = expr NEWLINE expr = ('-' term / expr '+' term / term) @@ -23,66 +23,112 @@ func TestLeftRrecursive(t *testing.T) { ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } haveLeftRecursion, err := builder.PrepareGrammar(grammar) - require.NoError(t, err) - require.True(t, haveLeftRecursion) + if err != nil { + t.Fatal(err) + } + if !haveLeftRecursion { + t.Fatalf("Recursion not found") + } + mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } - assert.False(t, mapRules["start"].LeftRecursive) - assert.True(t, mapRules["expr"].LeftRecursive) - assert.False(t, mapRules["term"].LeftRecursive) - assert.False(t, mapRules["foo"].LeftRecursive) - assert.False(t, mapRules["bar"].LeftRecursive) - assert.False(t, mapRules["baz"].LeftRecursive) + if mapRules["start"].LeftRecursive { + t.Fail() + } + if !mapRules["expr"].LeftRecursive { + t.Fail() + } + if mapRules["term"].LeftRecursive { + t.Fail() + } + if mapRules["foo"].LeftRecursive { + t.Fail() + } + if mapRules["bar"].LeftRecursive { + t.Fail() + } + if mapRules["baz"].LeftRecursive { + t.Fail() + } } func TestNullable(t *testing.T) { t.Parallel() + text := ` start = sign NUMBER sign = ('-' / '+')? ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } haveLeftRecursion, err := builder.PrepareGrammar(grammar) - require.NoError(t, err) - require.False(t, haveLeftRecursion) + if err != nil { + t.Fatal(err) + } + if haveLeftRecursion { + t.Fatalf("Recursion found") + } mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } - assert.False(t, mapRules["start"].Nullable) - assert.True(t, mapRules["sign"].Nullable) + if mapRules["start"].Nullable { + t.Fail() + } + if !mapRules["sign"].Nullable { + t.Fail() + } } -func TestAdvancedLeftRrecursive(t *testing.T) { +func TestAdvancedLeftRecursive(t *testing.T) { t.Parallel() + text := ` start = NUMBER / sign start sign = '-'? ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } haveLeftRecursion, err := builder.PrepareGrammar(grammar) - require.NoError(t, err) - require.True(t, haveLeftRecursion) + if err != nil { + t.Fatal(err) + } + if !haveLeftRecursion { + t.Fatalf("Recursion not found") + } mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } - assert.False(t, mapRules["start"].Nullable) - assert.True(t, mapRules["sign"].Nullable) - assert.True(t, mapRules["start"].LeftRecursive) - assert.False(t, mapRules["sign"].LeftRecursive) + if mapRules["start"].Nullable { + t.Fail() + } + if !mapRules["sign"].Nullable { + t.Fail() + } + if !mapRules["start"].LeftRecursive { + t.Fail() + } + if mapRules["sign"].LeftRecursive { + t.Fail() + } } -func TestMutuallyLeftRrecursive(t *testing.T) { +func TestMutuallyLeftRecursive(t *testing.T) { t.Parallel() + text := ` start = foo 'E' foo = bar 'A' / 'B' @@ -90,21 +136,34 @@ func TestMutuallyLeftRrecursive(t *testing.T) { ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } haveLeftRecursion, err := builder.PrepareGrammar(grammar) - require.NoError(t, err) - require.True(t, haveLeftRecursion) + if err != nil { + t.Fatal(err) + } + if !haveLeftRecursion { + t.Fatalf("Recursion not found") + } mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } - assert.False(t, mapRules["start"].LeftRecursive) - assert.True(t, mapRules["foo"].LeftRecursive) - assert.True(t, mapRules["bar"].LeftRecursive) + if mapRules["start"].LeftRecursive { + t.Fail() + } + if !mapRules["foo"].LeftRecursive { + t.Fail() + } + if !mapRules["bar"].LeftRecursive { + t.Fail() + } } -func TestNastyMutuallyLeftRrecursive(t *testing.T) { +func TestNastyMutuallyLeftRecursive(t *testing.T) { t.Parallel() + text := ` start = target '=' target = maybe '+' / NAME @@ -112,21 +171,34 @@ func TestNastyMutuallyLeftRrecursive(t *testing.T) { ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } haveLeftRecursion, err := builder.PrepareGrammar(grammar) - require.NoError(t, err) - require.True(t, haveLeftRecursion) + if err != nil { + t.Fatal(err) + } + if !haveLeftRecursion { + t.Fatalf("Recursion not found") + } mapRules := make(map[string]*ast.Rule, len(grammar.Rules)) for _, rule := range grammar.Rules { mapRules[rule.Name.Val] = rule } - assert.False(t, mapRules["start"].LeftRecursive) - assert.True(t, mapRules["target"].LeftRecursive) - assert.True(t, mapRules["maybe"].LeftRecursive) + if mapRules["start"].LeftRecursive { + t.Fail() + } + if !mapRules["target"].LeftRecursive { + t.Fail() + } + if !mapRules["maybe"].LeftRecursive { + t.Fail() + } } func TestLeftRecursionTooComplex(t *testing.T) { t.Parallel() + text := ` start = foo foo = bar '+' / baz '+' / '+' @@ -135,7 +207,11 @@ func TestLeftRecursionTooComplex(t *testing.T) { ` p := bootstrap.NewParser() grammar, err := p.Parse("", strings.NewReader(text)) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } _, err = builder.PrepareGrammar(grammar) - require.ErrorIs(t, err, builder.ErrNoLeader) + if !errors.Is(err, builder.ErrNoLeader) { + t.Fatalf("Got %s, but expected %s", err, builder.ErrNoLeader) + } } diff --git a/builder/scc.go b/builder/scc.go index 2de71eae..111a8ce6 100644 --- a/builder/scc.go +++ b/builder/scc.go @@ -10,7 +10,7 @@ func min(a1 int, a2 int) int { } // StronglyConnectedComponents compute strongly сonnected сomponents of a graph. -// Tarjan's strongly connected components algorithm +// Tarjan's strongly connected components algorithm. func StronglyConnectedComponents( vertices []string, edges map[string]map[string]bool, ) []map[string]bool { diff --git a/builder/scc_test.go b/builder/scc_test.go index 71161434..9af37818 100644 --- a/builder/scc_test.go +++ b/builder/scc_test.go @@ -1,12 +1,129 @@ package builder_test import ( + "bytes" + "reflect" "testing" "github.com/mna/pigeon/builder" - "github.com/stretchr/testify/require" ) +// isEmpty gets whether the specified object is considered empty or not. +func isEmpty(object interface{}) bool { + // get nil case out of the way + if object == nil { + return true + } + + objValue := reflect.ValueOf(object) + + switch objValue.Kind() { + // collection types are empty when they have no element + case reflect.Chan, reflect.Map, reflect.Slice: + return objValue.Len() == 0 + // pointers are empty if nil or if the value they point to is empty + case reflect.Ptr: + if objValue.IsNil() { + return true + } + deref := objValue.Elem().Interface() + return isEmpty(deref) + // for all other types, compare against the zero value + // array types are empty when they match their zero-initialized state + default: + zero := reflect.Zero(objValue.Type()) + return reflect.DeepEqual(object, zero.Interface()) + } +} + +// isList checks that the provided value is array or slice. +func isList(list interface{}) (ok bool) { + kind := reflect.TypeOf(list).Kind() + return kind == reflect.Array || kind == reflect.Slice +} + +// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. +// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and +// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. +func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { + aValue := reflect.ValueOf(listA) + bValue := reflect.ValueOf(listB) + + aLen := aValue.Len() + bLen := bValue.Len() + + // Mark indexes in bValue that we already used + visited := make([]bool, bLen) + for i := 0; i < aLen; i++ { + element := aValue.Index(i).Interface() + found := false + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + if ObjectsAreEqual(bValue.Index(j).Interface(), element) { + visited[j] = true + found = true + break + } + } + if !found { + extraA = append(extraA, element) + } + } + + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + extraB = append(extraB, bValue.Index(j).Interface()) + } + + return +} + +// ObjectsAreEqual determines if two objects are considered equal. +// +// This function does no assertion of any kind. +func ObjectsAreEqual(expected, actual interface{}) bool { + if expected == nil || actual == nil { + return expected == actual + } + + exp, ok := expected.([]byte) + if !ok { + return reflect.DeepEqual(expected, actual) + } + + act, ok := actual.([]byte) + if !ok { + return false + } + if exp == nil || act == nil { + return exp == nil && act == nil + } + return bytes.Equal(exp, act) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]). +func ElementsMatch(listA interface{}, listB interface{}) bool { + if isEmpty(listA) && isEmpty(listB) { + return true + } + + if !isList(listA) || !isList(listB) { + return false + } + + extraA, extraB := diffLists(listA, listB) + + return len(extraA) == 0 && len(extraB) == 0 +} + func TestStronglyConnectedComponents(t *testing.T) { //nolint:funlen t.Parallel() @@ -104,8 +221,10 @@ func TestStronglyConnectedComponents(t *testing.T) { //nolint:funlen for k := range testCase.graph { vertices = append(vertices, k) } - require.ElementsMatch(t, builder.StronglyConnectedComponents( - vertices, testCase.graph), testCase.want.sccs) + sccs := builder.StronglyConnectedComponents(vertices, testCase.graph) + if !ElementsMatch(sccs, testCase.want.sccs) { + t.FailNow() + } }) } } @@ -207,9 +326,11 @@ func TestFindCyclesInSCC(t *testing.T) { //nolint:funlen testCase := testCase t.Run(testCase.name, func(t *testing.T) { t.Parallel() - require.ElementsMatch(t, builder.FindCyclesInSCC( - testCase.graph, testCase.scc, testCase.start), - testCase.want.paths) + paths := builder.FindCyclesInSCC( + testCase.graph, testCase.scc, testCase.start) + if !ElementsMatch(paths, testCase.want.paths) { + t.FailNow() + } }) } } diff --git a/examples/calculator/calculator.go b/examples/calculator/calculator.go index d1a1a53c..5b3be589 100644 --- a/examples/calculator/calculator.go +++ b/examples/calculator/calculator.go @@ -99,8 +99,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Expr", @@ -158,8 +156,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Term", @@ -209,8 +205,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Factor", @@ -261,8 +255,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "AddOp", @@ -288,8 +280,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MulOp", @@ -315,8 +305,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Integer", @@ -349,8 +337,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -366,8 +352,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -378,8 +362,6 @@ var g = &grammar{ line: 101, col: 9, offset: 1916, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/examples/indentation/indentation.go b/examples/indentation/indentation.go index d4bf0a1c..97934cdf 100644 --- a/examples/indentation/indentation.go +++ b/examples/indentation/indentation.go @@ -62,8 +62,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Statements", @@ -83,8 +81,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Line", @@ -110,8 +106,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ReturnOp", @@ -147,8 +141,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Statement", @@ -239,8 +231,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Assignment", @@ -290,8 +280,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LogicalExpression", @@ -308,8 +296,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "AdditiveExpression", @@ -359,8 +345,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrimaryExpression", @@ -386,8 +370,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Integer", @@ -406,8 +388,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Identifier", @@ -438,8 +418,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "AddOp", @@ -465,8 +443,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -481,8 +457,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOL", @@ -539,8 +513,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Comment", @@ -566,8 +538,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -578,8 +548,6 @@ var g = &grammar{ line: 44, col: 8, offset: 1832, }, }, - leader: false, - leftRecursive: false, }, { name: "INDENTATION", @@ -606,8 +574,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "INDENT", @@ -616,8 +582,6 @@ var g = &grammar{ pos: position{line: 48, col: 10, offset: 1948}, run: (*parser).callonINDENT1, }, - leader: false, - leftRecursive: false, }, { name: "DEDENT", @@ -626,8 +590,6 @@ var g = &grammar{ pos: position{line: 50, col: 10, offset: 2035}, run: (*parser).callonDEDENT1, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/examples/json/json.go b/examples/json/json.go index 62d49635..2c3e5f63 100644 --- a/examples/json/json.go +++ b/examples/json/json.go @@ -59,8 +59,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Value", @@ -111,8 +109,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Object", @@ -216,8 +212,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Array", @@ -285,8 +279,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Number", @@ -341,8 +333,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Integer", @@ -374,8 +364,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Exponent", @@ -408,8 +396,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "String", @@ -473,8 +459,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EscapedChar", @@ -487,8 +471,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "EscapeSequence", @@ -506,8 +488,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharEscape", @@ -519,8 +499,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "UnicodeEscape", @@ -552,8 +530,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DecimalDigit", @@ -565,8 +541,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "NonZeroDecimalDigit", @@ -578,8 +552,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "HexDigit", @@ -591,8 +563,6 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "Bool", @@ -622,8 +592,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Null", @@ -638,8 +606,6 @@ var g = &grammar{ want: "\"null\"", }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -655,8 +621,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -667,8 +631,6 @@ var g = &grammar{ line: 89, col: 8, offset: 2077, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/examples/json/optimized-grammar/json.go b/examples/json/optimized-grammar/json.go index 04962df7..502b005e 100644 --- a/examples/json/optimized-grammar/json.go +++ b/examples/json/optimized-grammar/json.go @@ -67,8 +67,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Value", @@ -358,8 +356,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Object", @@ -711,8 +707,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Array", @@ -792,8 +786,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/examples/json/optimized/json.go b/examples/json/optimized/json.go index aee781ff..5815c14f 100644 --- a/examples/json/optimized/json.go +++ b/examples/json/optimized/json.go @@ -58,8 +58,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Value", @@ -110,8 +108,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Object", @@ -215,8 +211,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Array", @@ -284,8 +278,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Number", @@ -340,8 +332,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Integer", @@ -373,8 +363,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Exponent", @@ -408,8 +396,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "String", @@ -473,8 +459,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EscapedChar", @@ -488,8 +472,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "EscapeSequence", @@ -507,8 +489,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharEscape", @@ -521,8 +501,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "UnicodeEscape", @@ -554,8 +532,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DecimalDigit", @@ -568,8 +544,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "NonZeroDecimalDigit", @@ -582,8 +556,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "HexDigit", @@ -596,8 +568,6 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "Bool", @@ -627,8 +597,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Null", @@ -643,8 +611,6 @@ var g = &grammar{ want: "\"null\"", }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -661,8 +627,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -673,8 +637,6 @@ var g = &grammar{ line: 89, col: 8, offset: 2077, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/go.mod b/go.mod index 5cdce67a..b260a3d0 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,4 @@ require golang.org/x/tools v0.10.0 require ( golang.org/x/mod v0.11.0 // indirect golang.org/x/sys v0.9.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3fe54c21..1b85af0a 100644 --- a/go.sum +++ b/go.sum @@ -4,30 +4,3 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= - diff --git a/pigeon.go b/pigeon.go index 98072648..10a23f1d 100644 --- a/pigeon.go +++ b/pigeon.go @@ -81,8 +81,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Initializer", @@ -108,8 +106,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Rule", @@ -175,8 +171,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Expression", @@ -185,8 +179,6 @@ var g = &grammar{ pos: position{line: 41, col: 14, offset: 962}, name: "RecoveryExpr", }, - leader: false, - leftRecursive: false, }, { name: "RecoveryExpr", @@ -256,8 +248,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Labels", @@ -309,8 +299,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ChoiceExpr", @@ -362,8 +350,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ActionExpr", @@ -405,8 +391,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SeqExpr", @@ -448,8 +432,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LabeledExpr", @@ -506,8 +488,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrefixedExpr", @@ -550,8 +530,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrefixedOp", @@ -577,8 +555,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SuffixedExpr", @@ -621,8 +597,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SuffixedOp", @@ -654,8 +628,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "PrimaryExpr", @@ -722,8 +694,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RuleRefExpr", @@ -777,8 +747,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SemanticPredExpr", @@ -812,8 +780,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SemanticPredOp", @@ -845,8 +811,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RuleDefOp", @@ -880,8 +844,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SourceChar", @@ -889,8 +851,6 @@ var g = &grammar{ expr: &anyMatcher{ line: 193, col: 14, offset: 5208, }, - leader: false, - leftRecursive: false, }, { name: "Comment", @@ -908,8 +868,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MultiLineComment", @@ -952,8 +910,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MultiLineCommentNoLineTerminator", @@ -1005,8 +961,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleLineComment", @@ -1050,8 +1004,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Identifier", @@ -1068,8 +1020,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "IdentifierName", @@ -1094,8 +1044,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "IdentifierStart", @@ -1108,8 +1056,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "IdentifierPart", @@ -1130,8 +1076,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LitMatcher", @@ -1166,8 +1110,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "StringLiteral", @@ -1348,8 +1290,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DoubleStringChar", @@ -1407,8 +1347,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleStringChar", @@ -1466,8 +1404,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "RawStringChar", @@ -1490,8 +1426,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "DoubleStringEscape", @@ -1537,8 +1471,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleStringEscape", @@ -1584,8 +1516,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CommonEscapeSequence", @@ -1615,8 +1545,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharEscape", @@ -1674,8 +1602,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OctalEscape", @@ -1732,8 +1658,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "HexEscape", @@ -1794,8 +1718,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "LongUnicodeEscape", @@ -1884,8 +1806,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ShortUnicodeEscape", @@ -1958,8 +1878,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OctalDigit", @@ -1971,8 +1889,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "DecimalDigit", @@ -1984,8 +1900,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "HexDigit", @@ -1997,8 +1911,6 @@ var g = &grammar{ ignoreCase: true, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "CharClassMatcher", @@ -2116,8 +2028,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ClassCharRange", @@ -2141,8 +2051,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ClassChar", @@ -2200,8 +2108,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CharClassEscape", @@ -2261,8 +2167,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "UnicodeClassEscape", @@ -2389,8 +2293,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "SingleCharUnicodeClass", @@ -2402,8 +2304,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "AnyMatcher", @@ -2418,8 +2318,6 @@ var g = &grammar{ want: "\".\"", }, }, - leader: false, - leftRecursive: false, }, { name: "ThrowExpr", @@ -2493,8 +2391,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CodeBlock", @@ -2552,8 +2448,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Code", @@ -2618,8 +2512,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "__", @@ -2644,8 +2536,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -2666,8 +2556,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Whitespace", @@ -2679,8 +2567,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "EOL", @@ -2691,8 +2577,6 @@ var g = &grammar{ ignoreCase: false, want: "\"\\n\"", }, - leader: false, - leftRecursive: false, }, { name: "EOS", @@ -2750,8 +2634,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -2762,8 +2644,6 @@ var g = &grammar{ line: 334, col: 8, offset: 10147, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/alternate_entrypoint/altentry.go b/test/alternate_entrypoint/altentry.go index 895b8ec3..4ec3b848 100644 --- a/test/alternate_entrypoint/altentry.go +++ b/test/alternate_entrypoint/altentry.go @@ -59,8 +59,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Entry2", @@ -102,8 +100,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Entry3", @@ -136,8 +132,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "C", @@ -155,8 +149,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/andnot/andnot.go b/test/andnot/andnot.go index 32ff1d2b..f9439e00 100644 --- a/test/andnot/andnot.go +++ b/test/andnot/andnot.go @@ -52,8 +52,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "AB", @@ -90,8 +88,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "CD", @@ -119,8 +115,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -135,8 +129,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -147,8 +139,6 @@ var g = &grammar{ line: 20, col: 8, offset: 388, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/emptystate/emptystate.go b/test/emptystate/emptystate.go index aad4eeda..6096684f 100644 --- a/test/emptystate/emptystate.go +++ b/test/emptystate/emptystate.go @@ -55,8 +55,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "a", @@ -71,8 +69,6 @@ var g = &grammar{ want: "\"a\"", }, }, - leader: false, - leftRecursive: false, }, { name: "b", @@ -92,8 +88,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "c", @@ -108,8 +102,6 @@ var g = &grammar{ want: "\"c\"", }, }, - leader: false, - leftRecursive: false, }, { name: "d", @@ -129,8 +121,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "e", @@ -145,8 +135,6 @@ var g = &grammar{ want: "\"e\"", }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/errorpos/errorpos.go b/test/errorpos/errorpos.go index a93fe8d0..babc89cd 100644 --- a/test/errorpos/errorpos.go +++ b/test/errorpos/errorpos.go @@ -84,8 +84,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case01", @@ -134,8 +132,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case02", @@ -165,8 +161,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case03", @@ -202,8 +196,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case04", @@ -251,8 +243,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case05", @@ -287,8 +277,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case06", @@ -324,8 +312,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case07", @@ -380,8 +366,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case08", @@ -413,8 +397,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case09", @@ -447,8 +429,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case10", @@ -506,8 +486,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case11", @@ -545,8 +523,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "increment", @@ -557,8 +533,6 @@ var g = &grammar{ ignoreCase: false, want: "\"inc\"", }, - leader: false, - leftRecursive: false, }, { name: "decrement", @@ -569,8 +543,6 @@ var g = &grammar{ ignoreCase: false, want: "\"dec\"", }, - leader: false, - leftRecursive: false, }, { name: "zero", @@ -581,8 +553,6 @@ var g = &grammar{ ignoreCase: false, want: "\"zero\"", }, - leader: false, - leftRecursive: false, }, { name: "oneOrMore", @@ -593,8 +563,6 @@ var g = &grammar{ ignoreCase: false, want: "\"oneOrMore\"", }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -609,8 +577,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "__", @@ -622,8 +588,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -634,8 +598,6 @@ var g = &grammar{ line: 25, col: 8, offset: 725, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/global_store/global_store.go b/test/global_store/global_store.go index 208c1644..ebf25648 100644 --- a/test/global_store/global_store.go +++ b/test/global_store/global_store.go @@ -59,8 +59,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "increment", @@ -80,8 +78,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "decrement", @@ -101,8 +97,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "zero", @@ -122,8 +116,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -134,8 +126,6 @@ var g = &grammar{ line: 10, col: 8, offset: 473, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/goto/goto.go b/test/goto/goto.go index cf769282..ed0da62b 100644 --- a/test/goto/goto.go +++ b/test/goto/goto.go @@ -68,8 +68,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Line", @@ -112,8 +110,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Instruction", @@ -155,8 +151,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Label", @@ -184,8 +178,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "labelIdentifier", @@ -216,8 +208,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Noop", @@ -232,8 +222,6 @@ var g = &grammar{ want: "\"noop\"", }, }, - leader: false, - leftRecursive: false, }, { name: "Jump", @@ -265,8 +253,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "nl", @@ -282,8 +268,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "__", @@ -299,8 +283,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -316,8 +298,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -328,8 +308,6 @@ var g = &grammar{ line: 66, col: 8, offset: 1339, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/goto_state/goto_state.go b/test/goto_state/goto_state.go index 1fb6ee0f..ec8f1cc5 100644 --- a/test/goto_state/goto_state.go +++ b/test/goto_state/goto_state.go @@ -72,8 +72,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Line", @@ -116,8 +114,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Instruction", @@ -159,8 +155,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Label", @@ -188,8 +182,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "labelIdentifier", @@ -220,8 +212,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Noop", @@ -236,8 +226,6 @@ var g = &grammar{ want: "\"noop\"", }, }, - leader: false, - leftRecursive: false, }, { name: "Jump", @@ -273,8 +261,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "nl", @@ -290,8 +276,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "__", @@ -307,8 +291,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -324,8 +306,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -336,8 +316,6 @@ var g = &grammar{ line: 60, col: 8, offset: 1463, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_1/issue_1.go b/test/issue_1/issue_1.go index 1b0d9779..48714358 100644 --- a/test/issue_1/issue_1.go +++ b/test/issue_1/issue_1.go @@ -61,8 +61,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ID", @@ -81,8 +79,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_18/issue_18.go b/test/issue_18/issue_18.go index 98275fb8..50a0730f 100644 --- a/test/issue_18/issue_18.go +++ b/test/issue_18/issue_18.go @@ -80,8 +80,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "X", @@ -93,8 +91,6 @@ var g = &grammar{ ignoreCase: false, inverted: false, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -110,8 +106,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -122,8 +116,6 @@ var g = &grammar{ line: 38, col: 9, offset: 403, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_65/issue_65.go b/test/issue_65/issue_65.go index 0ad4b311..012ac913 100644 --- a/test/issue_65/issue_65.go +++ b/test/issue_65/issue_65.go @@ -37,8 +37,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "List", @@ -70,8 +68,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "X", @@ -94,8 +90,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Y", @@ -110,8 +104,6 @@ var g = &grammar{ want: "\"Y\"", }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_65/optimized-grammar/issue_65.go b/test/issue_65/optimized-grammar/issue_65.go index ce8ba010..56bcd080 100644 --- a/test/issue_65/optimized-grammar/issue_65.go +++ b/test/issue_65/optimized-grammar/issue_65.go @@ -79,8 +79,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_65/optimized/issue_65.go b/test/issue_65/optimized/issue_65.go index 015691f1..2d09c6a4 100644 --- a/test/issue_65/optimized/issue_65.go +++ b/test/issue_65/optimized/issue_65.go @@ -36,8 +36,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "List", @@ -69,8 +67,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "X", @@ -93,8 +89,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Y", @@ -109,8 +103,6 @@ var g = &grammar{ want: "\"Y\"", }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_70/issue_70.go b/test/issue_70/issue_70.go index 5f841f09..57662dbf 100644 --- a/test/issue_70/issue_70.go +++ b/test/issue_70/issue_70.go @@ -45,8 +45,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Y", @@ -55,8 +53,6 @@ var g = &grammar{ pos: position{line: 6, col: 5, offset: 58}, name: "Z", }, - leader: false, - leftRecursive: false, }, { name: "Z", @@ -74,8 +70,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_70/optimized-grammar/issue_70.go b/test/issue_70/optimized-grammar/issue_70.go index b7a5527f..c6abac26 100644 --- a/test/issue_70/optimized-grammar/issue_70.go +++ b/test/issue_70/optimized-grammar/issue_70.go @@ -54,8 +54,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_70/optimized/issue_70.go b/test/issue_70/optimized/issue_70.go index 7e535458..a1c30e98 100644 --- a/test/issue_70/optimized/issue_70.go +++ b/test/issue_70/optimized/issue_70.go @@ -44,8 +44,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Y", @@ -54,8 +52,6 @@ var g = &grammar{ pos: position{line: 6, col: 5, offset: 58}, name: "Z", }, - leader: false, - leftRecursive: false, }, { name: "Z", @@ -73,8 +69,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/issue_80/issue_80.go b/test/issue_80/issue_80.go index 35ca8aec..2275dcd3 100644 --- a/test/issue_80/issue_80.go +++ b/test/issue_80/issue_80.go @@ -73,8 +73,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -85,8 +83,6 @@ var g = &grammar{ line: 17, col: 8, offset: 276, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/labeled_failures/labeled_failures.go b/test/labeled_failures/labeled_failures.go index aa7f898c..38f6f63c 100644 --- a/test/labeled_failures/labeled_failures.go +++ b/test/labeled_failures/labeled_failures.go @@ -80,8 +80,6 @@ var g = &grammar{ "errId", }, }, - leader: false, - leftRecursive: false, }, { name: "List", @@ -126,8 +124,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ID", @@ -164,8 +160,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Comma", @@ -194,8 +188,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Sp", @@ -210,8 +202,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrComma", @@ -249,8 +239,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrID", @@ -288,8 +276,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/left_recursion.go index 8c9aec4d..8a58d2d2 100644 --- a/test/left_recursion/left_recursion.go +++ b/test/left_recursion/left_recursion.go @@ -104,13 +104,13 @@ var g = &grammar{ want: "\"*\"", }, &litMatcher{ - pos: position{line: 11, col: 20, offset: 167}, + pos: position{line: 11, col: 22, offset: 169}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 11, col: 24, offset: 171}, + pos: position{line: 11, col: 28, offset: 175}, val: "%", ignoreCase: false, want: "\"%\"", @@ -119,10 +119,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 11, col: 29, offset: 176}, + pos: position{line: 11, col: 33, offset: 180}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 11, col: 31, offset: 178}, + pos: position{line: 11, col: 35, offset: 182}, name: "expr", }, }, @@ -130,33 +130,33 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 16, col: 5, offset: 317}, + pos: position{line: 16, col: 5, offset: 321}, run: (*parser).callonexpr18, expr: &seqExpr{ - pos: position{line: 16, col: 5, offset: 317}, + pos: position{line: 16, col: 5, offset: 321}, exprs: []any{ &labeledExpr{ - pos: position{line: 16, col: 5, offset: 317}, + pos: position{line: 16, col: 5, offset: 321}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 16, col: 7, offset: 319}, + pos: position{line: 16, col: 7, offset: 323}, name: "expr", }, }, &labeledExpr{ - pos: position{line: 16, col: 12, offset: 324}, + pos: position{line: 16, col: 12, offset: 328}, label: "op", expr: &choiceExpr{ - pos: position{line: 16, col: 16, offset: 328}, + pos: position{line: 16, col: 16, offset: 332}, alternatives: []any{ &litMatcher{ - pos: position{line: 16, col: 16, offset: 328}, + pos: position{line: 16, col: 16, offset: 332}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 16, col: 20, offset: 332}, + pos: position{line: 16, col: 22, offset: 338}, val: "-", ignoreCase: false, want: "\"-\"", @@ -165,10 +165,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 16, col: 25, offset: 337}, + pos: position{line: 16, col: 27, offset: 343}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 16, col: 27, offset: 339}, + pos: position{line: 16, col: 29, offset: 345}, name: "expr", }, }, @@ -176,10 +176,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 21, col: 5, offset: 477}, + pos: position{line: 21, col: 5, offset: 483}, run: (*parser).callonexpr28, expr: &ruleRefExpr{ - pos: position{line: 21, col: 5, offset: 477}, + pos: position{line: 21, col: 5, offset: 483}, name: "term", }, }, @@ -190,11 +190,11 @@ var g = &grammar{ }, { name: "term", - pos: position{line: 25, col: 1, offset: 518}, + pos: position{line: 25, col: 1, offset: 524}, expr: &oneOrMoreExpr{ - pos: position{line: 25, col: 8, offset: 525}, + pos: position{line: 25, col: 8, offset: 531}, expr: &charClassMatcher{ - pos: position{line: 25, col: 8, offset: 525}, + pos: position{line: 25, col: 8, offset: 531}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg index a8c2d9a2..cba32c74 100644 --- a/test/left_recursion/left_recursion.peg +++ b/test/left_recursion/left_recursion.peg @@ -8,12 +8,12 @@ start = a:expr !. { expr = '-' a:expr { strA := a.(string) return "(" + "-" + strA + ")", nil -} / a:expr op:('*'/'/'/'%') b:expr { +} / a:expr op:('*' / '/' / '%') b:expr { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil -} / a:expr op:('+'/'-') b:expr { +} / a:expr op:('+' / '-') b:expr { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) @@ -22,4 +22,4 @@ expr = '-' a:expr { return string(c.text), nil } -term = [0-9]+ \ No newline at end of file +term = [0-9]+ diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/left_recursion_test.go index cf9695ed..63412630 100644 --- a/test/left_recursion/left_recursion_test.go +++ b/test/left_recursion/left_recursion_test.go @@ -2,16 +2,25 @@ package leftrecursion import ( "testing" - - "github.com/stretchr/testify/require" ) func TestLeftRecursion(t *testing.T) { t.Parallel() + data := "7+10/2*-4+5*3%6-8*6" res, err := Parse("", []byte(data)) - require.NoError(t, err) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + data, err) + } str, ok := res.(string) - require.True(t, ok) - require.Equal(t, str, "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))") + if !ok { + t.FailNow() + } + want := "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))" + if str != want { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", data, str, want) + } } diff --git a/test/linear/linear.go b/test/linear/linear.go index 0c8c8982..2ac895cb 100644 --- a/test/linear/linear.go +++ b/test/linear/linear.go @@ -139,8 +139,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "L", @@ -155,8 +153,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "N", @@ -171,8 +167,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "S", @@ -187,8 +181,6 @@ var g = &grammar{ inverted: false, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -199,8 +191,6 @@ var g = &grammar{ line: 11, col: 8, offset: 187, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/max_expr_cnt/maxexpr.go b/test/max_expr_cnt/maxexpr.go index 37af9ae1..6fe474ee 100644 --- a/test/max_expr_cnt/maxexpr.go +++ b/test/max_expr_cnt/maxexpr.go @@ -26,8 +26,6 @@ var g = &grammar{ pos: position{line: 6, col: 14, offset: 66}, name: "long_rule2", }, - leader: false, - leftRecursive: false, }, { name: "long_rule2", @@ -36,8 +34,6 @@ var g = &grammar{ pos: position{line: 7, col: 14, offset: 90}, name: "long_rule3", }, - leader: false, - leftRecursive: false, }, { name: "long_rule3", @@ -46,8 +42,6 @@ var g = &grammar{ pos: position{line: 8, col: 14, offset: 114}, name: "long_rule4", }, - leader: false, - leftRecursive: false, }, { name: "long_rule4", @@ -56,8 +50,6 @@ var g = &grammar{ pos: position{line: 9, col: 14, offset: 138}, name: "long_rule5", }, - leader: false, - leftRecursive: false, }, { name: "long_rule5", @@ -66,40 +58,16 @@ var g = &grammar{ pos: position{line: 10, col: 14, offset: 162}, name: "long_rule6", }, - leader: false, - leftRecursive: false, }, { name: "long_rule6", pos: position{line: 11, col: 1, offset: 173}, - expr: &ruleRefExpr{ - pos: position{line: 11, col: 14, offset: 186}, - name: "long_rule7", - }, - leader: false, - leftRecursive: false, - }, - { - name: "long_rule7", - pos: position{line: 12, col: 1, offset: 197}, - expr: &ruleRefExpr{ - pos: position{line: 12, col: 14, offset: 210}, - name: "long_rule8", - }, - leader: false, - leftRecursive: false, - }, - { - name: "long_rule8", - pos: position{line: 13, col: 1, offset: 221}, expr: &litMatcher{ - pos: position{line: 13, col: 14, offset: 234}, + pos: position{line: 11, col: 14, offset: 186}, val: " ", ignoreCase: false, want: "\" \"", }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/max_expr_cnt/maxexpr.peg b/test/max_expr_cnt/maxexpr.peg index 4b4558aa..c3e9dc82 100644 --- a/test/max_expr_cnt/maxexpr.peg +++ b/test/max_expr_cnt/maxexpr.peg @@ -8,6 +8,4 @@ long_rule2 = long_rule3 long_rule3 = long_rule4 long_rule4 = long_rule5 long_rule5 = long_rule6 -long_rule6 = long_rule7 -long_rule7 = long_rule8 -long_rule8 = " " \ No newline at end of file +long_rule6 = " " \ No newline at end of file diff --git a/test/predicates/predicates.go b/test/predicates/predicates.go index 6cf95c2f..5e351a7c 100644 --- a/test/predicates/predicates.go +++ b/test/predicates/predicates.go @@ -84,8 +84,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "B", @@ -139,8 +137,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "C", @@ -176,8 +172,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/runeerror/runeerror.go b/test/runeerror/runeerror.go index c2fd891b..151bc152 100644 --- a/test/runeerror/runeerror.go +++ b/test/runeerror/runeerror.go @@ -78,8 +78,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/state/state.go b/test/state/state.go index 0223a5d1..5f25c5e8 100644 --- a/test/state/state.go +++ b/test/state/state.go @@ -115,8 +115,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/stateclone/stateclone.go b/test/stateclone/stateclone.go index 7c0c44b3..cea5e3b6 100644 --- a/test/stateclone/stateclone.go +++ b/test/stateclone/stateclone.go @@ -75,8 +75,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "x", @@ -102,8 +100,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "y", @@ -129,8 +125,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "z", @@ -150,8 +144,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "c", @@ -171,8 +163,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "bc", @@ -192,8 +182,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ws", @@ -215,8 +203,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/statereadonly/statereadonly.go b/test/statereadonly/statereadonly.go index c78e5858..38cd8f62 100644 --- a/test/statereadonly/statereadonly.go +++ b/test/statereadonly/statereadonly.go @@ -67,8 +67,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "x", @@ -94,8 +92,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "y", @@ -121,8 +117,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "z", @@ -137,8 +131,6 @@ var g = &grammar{ want: "\"abcf\"", }, }, - leader: false, - leftRecursive: false, }, { name: "c", @@ -158,8 +150,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "bc", @@ -179,8 +169,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ws", @@ -202,8 +190,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/staterestore/optimized/staterestore.go b/test/staterestore/optimized/staterestore.go index 75bb0f2d..8909cedc 100644 --- a/test/staterestore/optimized/staterestore.go +++ b/test/staterestore/optimized/staterestore.go @@ -356,8 +356,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestAnd", @@ -402,8 +400,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestNot", @@ -459,8 +455,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/staterestore/standard/staterestore.go b/test/staterestore/standard/staterestore.go index b07459b3..77dd3825 100644 --- a/test/staterestore/standard/staterestore.go +++ b/test/staterestore/standard/staterestore.go @@ -62,8 +62,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestAnd", @@ -95,8 +93,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestNot", @@ -130,8 +126,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Z_", @@ -151,8 +145,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Expr", @@ -189,8 +181,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOL", @@ -210,8 +200,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Comment", @@ -245,8 +233,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -267,8 +253,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -279,8 +263,6 @@ var g = &grammar{ line: 46, col: 9, offset: 784, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/staterestore/staterestore.go b/test/staterestore/staterestore.go index b07459b3..77dd3825 100644 --- a/test/staterestore/staterestore.go +++ b/test/staterestore/staterestore.go @@ -62,8 +62,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestAnd", @@ -95,8 +93,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "TestNot", @@ -130,8 +126,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Z_", @@ -151,8 +145,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Expr", @@ -189,8 +181,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOL", @@ -210,8 +200,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "Comment", @@ -245,8 +233,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "_", @@ -267,8 +253,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "EOF", @@ -279,8 +263,6 @@ var g = &grammar{ line: 46, col: 9, offset: 784, }, }, - leader: false, - leftRecursive: false, }, }, } diff --git a/test/thrownrecover/thrownrecover.go b/test/thrownrecover/thrownrecover.go index e3a96858..f0cafa8a 100644 --- a/test/thrownrecover/thrownrecover.go +++ b/test/thrownrecover/thrownrecover.go @@ -43,8 +43,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case01", @@ -72,8 +70,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "MultiLabelRecover", @@ -93,8 +89,6 @@ var g = &grammar{ "errOther", }, }, - leader: false, - leftRecursive: false, }, { name: "number", @@ -135,8 +129,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "digit", @@ -188,8 +180,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrNonNumber", @@ -228,8 +218,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case02", @@ -258,8 +246,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ThrowUndefLabel", @@ -268,8 +254,6 @@ var g = &grammar{ pos: position{line: 33, col: 19, offset: 744}, label: "undeflabel", }, - leader: false, - leftRecursive: false, }, { name: "case03", @@ -297,8 +281,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OuterRecover03", @@ -328,8 +310,6 @@ var g = &grammar{ "errOther", }, }, - leader: false, - leftRecursive: false, }, { name: "InnerRecover03", @@ -348,8 +328,6 @@ var g = &grammar{ "errAlphaLower", }, }, - leader: false, - leftRecursive: false, }, { name: "number03", @@ -390,8 +368,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "digit03", @@ -470,8 +446,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrAlphaInner03", @@ -510,8 +484,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrAlphaOuter03", @@ -550,8 +522,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrOtherOuter03", @@ -590,8 +560,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "case04", @@ -619,8 +587,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "OuterRecover04", @@ -650,8 +616,6 @@ var g = &grammar{ "errOther", }, }, - leader: false, - leftRecursive: false, }, { name: "InnerRecover04", @@ -670,8 +634,6 @@ var g = &grammar{ "errAlphaLower", }, }, - leader: false, - leftRecursive: false, }, { name: "number04", @@ -712,8 +674,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "digit04", @@ -792,8 +752,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrAlphaInner04", @@ -802,8 +760,6 @@ var g = &grammar{ pos: position{line: 83, col: 19, offset: 2363}, run: (*parser).callonErrAlphaInner041, }, - leader: false, - leftRecursive: false, }, { name: "ErrAlphaOuter04", @@ -842,8 +798,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, { name: "ErrOtherOuter04", @@ -882,8 +836,6 @@ var g = &grammar{ }, }, }, - leader: false, - leftRecursive: false, }, }, } From 8a83e813e07d301e49df0453187acc51fa14ecad Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Sun, 25 Jun 2023 12:21:56 +0300 Subject: [PATCH 05/14] Add removal of parts needed only to support the left recursion --- .../cmd/bootstrap-pigeon/bootstrap_pigeon.go | 34 ++++--------------- builder/generated_static_code.go | 32 +++++++++++++++-- builder/static_code.go | 32 +++++++++++++++-- examples/calculator/calculator.go | 34 ++++--------------- examples/indentation/indentation.go | 34 ++++--------------- examples/json/json.go | 34 ++++--------------- examples/json/optimized-grammar/json.go | 34 ++++--------------- examples/json/optimized/json.go | 32 +++-------------- pigeon.go | 34 ++++--------------- test/alternate_entrypoint/altentry.go | 34 ++++--------------- test/andnot/andnot.go | 34 ++++--------------- test/emptystate/emptystate.go | 34 ++++--------------- test/errorpos/errorpos.go | 34 ++++--------------- test/global_store/global_store.go | 34 ++++--------------- test/goto/goto.go | 34 ++++--------------- test/goto_state/goto_state.go | 34 ++++--------------- test/issue_1/issue_1.go | 34 ++++--------------- test/issue_18/issue_18.go | 34 ++++--------------- test/issue_65/issue_65.go | 34 ++++--------------- test/issue_65/optimized-grammar/issue_65.go | 34 ++++--------------- test/issue_65/optimized/issue_65.go | 32 +++-------------- test/issue_70/issue_70.go | 34 ++++--------------- test/issue_70/optimized-grammar/issue_70.go | 34 ++++--------------- test/issue_70/optimized/issue_70.go | 32 +++-------------- test/issue_70b/issue_70b.go | 8 +++-- test/issue_80/issue_80.go | 34 ++++--------------- test/labeled_failures/labeled_failures.go | 34 ++++--------------- test/left_recursion/left_recursion.go | 8 +++-- test/linear/linear.go | 34 ++++--------------- test/max_expr_cnt/maxexpr.go | 34 ++++--------------- test/predicates/predicates.go | 34 ++++--------------- test/runeerror/runeerror.go | 34 ++++--------------- test/state/state.go | 34 ++++--------------- test/stateclone/stateclone.go | 34 ++++--------------- test/statereadonly/statereadonly.go | 34 ++++--------------- test/staterestore/optimized/staterestore.go | 32 +++-------------- test/staterestore/standard/staterestore.go | 34 ++++--------------- test/staterestore/staterestore.go | 34 ++++--------------- test/thrownrecover/thrownrecover.go | 34 ++++--------------- 39 files changed, 278 insertions(+), 984 deletions(-) diff --git a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go index 5ee1176e..27ee02e2 100644 --- a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go +++ b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go @@ -2473,9 +2473,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } type choiceExpr struct { @@ -2693,11 +2690,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - type parser struct { filename string pt savepoint @@ -2720,7 +2712,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -2774,27 +2766,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -2864,7 +2844,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -3157,7 +3137,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -3200,7 +3179,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -3344,7 +3322,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index e7ecf4af..b32c714c 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -252,8 +252,10 @@ type rule struct { displayName string expr any + // ==template== {{ if .LeftRecursion }} leader bool leftRecursive bool + // {{ end }} ==template== } // {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} @@ -492,11 +494,14 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +// ==template== {{ if .LeftRecursion }} type ruleWithExpsStack struct { rule *rule estack []any } +// {{ end }} ==template== + // {{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} type parser struct { filename string @@ -524,7 +529,11 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors + // ==template== {{ if .LeftRecursion }} rstack []ruleWithExpsStack + // {{ else }} + rstack []*rule + // {{ end }} ==template== // parse fail maxFailPos position @@ -578,13 +587,26 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { + // ==template== {{ if .LeftRecursion }} p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + // {{ else }} + p.rstack = append(p.rstack, rule) + // {{ end }} ==template== } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } +func (p *parser) getRule() *rule { + // ==template== {{ if .LeftRecursion }} + return p.rstack[len(p.rstack)-1].rule + // {{ else }} + return p.rstack[len(p.rstack)-1] + // {{ end }} ==template== +} + +// ==template== {{ if .LeftRecursion }} func (p *parser) pushExpr(expr any) { if len(p.rstack) == 0 { return @@ -601,6 +623,8 @@ func (p *parser) popExpr() { p.rstack[len(p.rstack)-1].estack)-1] } +// {{ end }} ==template== + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -671,7 +695,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1141,7 +1165,9 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + // ==template== {{ if .LeftRecursion }} p.pushExpr(expr) + // {{ end }} ==template== var val any var ok bool switch expr := expr.(type) { @@ -1186,7 +1212,9 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + // ==template== {{ if .LeftRecursion }} p.popExpr() + // {{ end }} ==template== return val, ok } @@ -1369,7 +1397,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/builder/static_code.go b/builder/static_code.go index 77d220cb..9b54c32c 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -270,8 +270,10 @@ type rule struct { displayName string expr any + // ==template== {{ if .LeftRecursion }} leader bool leftRecursive bool + // {{ end }} ==template== } // {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }} @@ -510,11 +512,14 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } +// ==template== {{ if .LeftRecursion }} type ruleWithExpsStack struct { rule *rule estack []any } +// {{ end }} ==template== + // {{ if .Nolint }} nolint: structcheck,maligned {{else}} ==template== {{ end }} type parser struct { filename string @@ -542,7 +547,11 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors + // ==template== {{ if .LeftRecursion }} rstack []ruleWithExpsStack + // {{ else }} + rstack []*rule + // {{ end }} ==template== // parse fail maxFailPos position @@ -596,13 +605,26 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { + // ==template== {{ if .LeftRecursion }} p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + // {{ else }} + p.rstack = append(p.rstack, rule) + // {{ end }} ==template== } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } +func (p *parser) getRule() *rule { + // ==template== {{ if .LeftRecursion }} + return p.rstack[len(p.rstack)-1].rule + // {{ else }} + return p.rstack[len(p.rstack)-1] + // {{ end }} ==template== +} + +// ==template== {{ if .LeftRecursion }} func (p *parser) pushExpr(expr any) { if len(p.rstack) == 0 { return @@ -619,6 +641,8 @@ func (p *parser) popExpr() { p.rstack[len(p.rstack)-1].estack)-1] } +// {{ end }} ==template== + // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -689,7 +713,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1159,7 +1183,9 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } + // ==template== {{ if .LeftRecursion }} p.pushExpr(expr) + // {{ end }} ==template== var val any var ok bool switch expr := expr.(type) { @@ -1204,7 +1230,9 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } + // ==template== {{ if .LeftRecursion }} p.popExpr() + // {{ end }} ==template== return val, ok } @@ -1387,7 +1415,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/calculator/calculator.go b/examples/calculator/calculator.go index 5b3be589..ed5147c7 100644 --- a/examples/calculator/calculator.go +++ b/examples/calculator/calculator.go @@ -691,9 +691,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -926,11 +923,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -954,7 +946,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1008,27 +1000,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1098,7 +1078,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1393,7 +1373,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1436,7 +1415,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1581,7 +1559,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/indentation/indentation.go b/examples/indentation/indentation.go index 97934cdf..82ce0254 100644 --- a/examples/indentation/indentation.go +++ b/examples/indentation/indentation.go @@ -1004,9 +1004,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1239,11 +1236,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1267,7 +1259,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1321,27 +1313,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1411,7 +1391,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1706,7 +1686,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1749,7 +1728,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1894,7 +1872,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/json.go b/examples/json/json.go index 2c3e5f63..c5b71757 100644 --- a/examples/json/json.go +++ b/examples/json/json.go @@ -986,9 +986,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1221,11 +1218,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1249,7 +1241,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1303,27 +1295,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1393,7 +1373,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1688,7 +1668,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1731,7 +1710,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1876,7 +1854,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized-grammar/json.go b/examples/json/optimized-grammar/json.go index 502b005e..f601a98a 100644 --- a/examples/json/optimized-grammar/json.go +++ b/examples/json/optimized-grammar/json.go @@ -1163,9 +1163,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1398,11 +1395,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1426,7 +1418,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1480,27 +1472,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1570,7 +1550,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1865,7 +1845,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1908,7 +1887,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -2053,7 +2031,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized/json.go b/examples/json/optimized/json.go index 5815c14f..dfde3d88 100644 --- a/examples/json/optimized/json.go +++ b/examples/json/optimized/json.go @@ -921,9 +921,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1149,11 +1146,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1171,7 +1163,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1225,27 +1217,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1290,7 +1270,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1467,7 +1447,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1508,7 +1487,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } diff --git a/pigeon.go b/pigeon.go index 10a23f1d..f54086be 100644 --- a/pigeon.go +++ b/pigeon.go @@ -3442,9 +3442,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -3677,11 +3674,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -3705,7 +3697,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -3759,27 +3751,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -3849,7 +3829,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -4144,7 +4124,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -4187,7 +4166,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -4332,7 +4310,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/alternate_entrypoint/altentry.go b/test/alternate_entrypoint/altentry.go index 4ec3b848..e2ad4f10 100644 --- a/test/alternate_entrypoint/altentry.go +++ b/test/alternate_entrypoint/altentry.go @@ -460,9 +460,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -695,11 +692,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -723,7 +715,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -777,27 +769,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -867,7 +847,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1162,7 +1142,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1205,7 +1184,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1350,7 +1328,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/andnot/andnot.go b/test/andnot/andnot.go index f9439e00..a6fa33ac 100644 --- a/test/andnot/andnot.go +++ b/test/andnot/andnot.go @@ -400,9 +400,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -635,11 +632,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -663,7 +655,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -717,27 +709,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -807,7 +787,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1102,7 +1082,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1145,7 +1124,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1290,7 +1268,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/emptystate/emptystate.go b/test/emptystate/emptystate.go index 6096684f..9bd4512d 100644 --- a/test/emptystate/emptystate.go +++ b/test/emptystate/emptystate.go @@ -460,9 +460,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -695,11 +692,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -723,7 +715,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -777,27 +769,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -867,7 +847,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1162,7 +1142,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1205,7 +1184,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1350,7 +1328,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/errorpos/errorpos.go b/test/errorpos/errorpos.go index babc89cd..87e47060 100644 --- a/test/errorpos/errorpos.go +++ b/test/errorpos/errorpos.go @@ -839,9 +839,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1074,11 +1071,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1102,7 +1094,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1156,27 +1148,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1246,7 +1226,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1541,7 +1521,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1584,7 +1563,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1729,7 +1707,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/global_store/global_store.go b/test/global_store/global_store.go index ebf25648..b08f8c3f 100644 --- a/test/global_store/global_store.go +++ b/test/global_store/global_store.go @@ -421,9 +421,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -656,11 +653,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -684,7 +676,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -738,27 +730,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -828,7 +808,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1123,7 +1103,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1166,7 +1145,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1311,7 +1289,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto/goto.go b/test/goto/goto.go index ed0da62b..63e3f3bc 100644 --- a/test/goto/goto.go +++ b/test/goto/goto.go @@ -638,9 +638,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -873,11 +870,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -901,7 +893,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -955,27 +947,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1045,7 +1025,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1340,7 +1320,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1383,7 +1362,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1528,7 +1506,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto_state/goto_state.go b/test/goto_state/goto_state.go index ec8f1cc5..3931234d 100644 --- a/test/goto_state/goto_state.go +++ b/test/goto_state/goto_state.go @@ -666,9 +666,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -901,11 +898,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -929,7 +921,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -983,27 +975,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1073,7 +1053,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1368,7 +1348,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1411,7 +1390,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1556,7 +1534,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_1/issue_1.go b/test/issue_1/issue_1.go index 48714358..0eefbe6a 100644 --- a/test/issue_1/issue_1.go +++ b/test/issue_1/issue_1.go @@ -340,9 +340,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -575,11 +572,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -603,7 +595,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -657,27 +649,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -747,7 +727,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1042,7 +1022,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1085,7 +1064,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1230,7 +1208,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_18/issue_18.go b/test/issue_18/issue_18.go index 50a0730f..d55f1a94 100644 --- a/test/issue_18/issue_18.go +++ b/test/issue_18/issue_18.go @@ -357,9 +357,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -592,11 +589,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -620,7 +612,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -674,27 +666,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -764,7 +744,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1059,7 +1039,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1102,7 +1081,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1247,7 +1225,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/issue_65.go b/test/issue_65/issue_65.go index 012ac913..05478369 100644 --- a/test/issue_65/issue_65.go +++ b/test/issue_65/issue_65.go @@ -356,9 +356,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -591,11 +588,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -619,7 +611,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -673,27 +665,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -763,7 +743,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1058,7 +1038,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1101,7 +1080,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1246,7 +1224,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized-grammar/issue_65.go b/test/issue_65/optimized-grammar/issue_65.go index 56bcd080..93aa0838 100644 --- a/test/issue_65/optimized-grammar/issue_65.go +++ b/test/issue_65/optimized-grammar/issue_65.go @@ -342,9 +342,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -577,11 +574,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -605,7 +597,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -659,27 +651,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -749,7 +729,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1044,7 +1024,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1087,7 +1066,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1232,7 +1210,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized/issue_65.go b/test/issue_65/optimized/issue_65.go index 2d09c6a4..818bd81e 100644 --- a/test/issue_65/optimized/issue_65.go +++ b/test/issue_65/optimized/issue_65.go @@ -284,9 +284,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -512,11 +509,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -534,7 +526,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -588,27 +580,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -653,7 +633,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -830,7 +810,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -871,7 +850,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } diff --git a/test/issue_70/issue_70.go b/test/issue_70/issue_70.go index 57662dbf..0c2791d6 100644 --- a/test/issue_70/issue_70.go +++ b/test/issue_70/issue_70.go @@ -331,9 +331,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -566,11 +563,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -594,7 +586,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -648,27 +640,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -738,7 +718,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1033,7 +1013,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1076,7 +1055,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1221,7 +1199,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized-grammar/issue_70.go b/test/issue_70/optimized-grammar/issue_70.go index c6abac26..b4179370 100644 --- a/test/issue_70/optimized-grammar/issue_70.go +++ b/test/issue_70/optimized-grammar/issue_70.go @@ -315,9 +315,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -550,11 +547,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -578,7 +570,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -632,27 +624,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -722,7 +702,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1017,7 +997,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1060,7 +1039,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1205,7 +1183,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized/issue_70.go b/test/issue_70/optimized/issue_70.go index a1c30e98..ba3e22ae 100644 --- a/test/issue_70/optimized/issue_70.go +++ b/test/issue_70/optimized/issue_70.go @@ -259,9 +259,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -487,11 +484,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -509,7 +501,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -563,27 +555,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -628,7 +608,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -805,7 +785,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -846,7 +825,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index f3863b0b..d5af0fb1 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -644,6 +644,10 @@ func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1].rule +} + func (p *parser) pushExpr(expr any) { if len(p.rstack) == 0 { return @@ -727,7 +731,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1325,7 +1329,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_80/issue_80.go b/test/issue_80/issue_80.go index 2275dcd3..170de2b9 100644 --- a/test/issue_80/issue_80.go +++ b/test/issue_80/issue_80.go @@ -350,9 +350,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -585,11 +582,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -613,7 +605,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -667,27 +659,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -757,7 +737,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1052,7 +1032,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1095,7 +1074,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1240,7 +1218,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/labeled_failures/labeled_failures.go b/test/labeled_failures/labeled_failures.go index 38f6f63c..5daf7058 100644 --- a/test/labeled_failures/labeled_failures.go +++ b/test/labeled_failures/labeled_failures.go @@ -579,9 +579,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -814,11 +811,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -842,7 +834,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -896,27 +888,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -986,7 +966,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1281,7 +1261,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1324,7 +1303,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1469,7 +1447,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/left_recursion.go index 8a58d2d2..a9959d11 100644 --- a/test/left_recursion/left_recursion.go +++ b/test/left_recursion/left_recursion.go @@ -825,6 +825,10 @@ func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1].rule +} + func (p *parser) pushExpr(expr any) { if len(p.rstack) == 0 { return @@ -908,7 +912,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1506,7 +1510,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/linear/linear.go b/test/linear/linear.go index 2ac895cb..af2efe89 100644 --- a/test/linear/linear.go +++ b/test/linear/linear.go @@ -432,9 +432,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -667,11 +664,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -695,7 +687,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -749,27 +741,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -839,7 +819,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1134,7 +1114,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1177,7 +1156,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1322,7 +1300,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/max_expr_cnt/maxexpr.go b/test/max_expr_cnt/maxexpr.go index 6fe474ee..834118ed 100644 --- a/test/max_expr_cnt/maxexpr.go +++ b/test/max_expr_cnt/maxexpr.go @@ -309,9 +309,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -544,11 +541,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -572,7 +564,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -626,27 +618,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -716,7 +696,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1011,7 +991,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1054,7 +1033,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1199,7 +1177,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/predicates/predicates.go b/test/predicates/predicates.go index 5e351a7c..de4f3e7f 100644 --- a/test/predicates/predicates.go +++ b/test/predicates/predicates.go @@ -486,9 +486,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -721,11 +718,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -749,7 +741,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -803,27 +795,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -893,7 +873,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1188,7 +1168,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1231,7 +1210,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1376,7 +1354,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/runeerror/runeerror.go b/test/runeerror/runeerror.go index 151bc152..b46bc6e0 100644 --- a/test/runeerror/runeerror.go +++ b/test/runeerror/runeerror.go @@ -331,9 +331,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -566,11 +563,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -594,7 +586,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -648,27 +640,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -738,7 +718,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1033,7 +1013,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1076,7 +1055,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1221,7 +1199,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/state/state.go b/test/state/state.go index 5f25c5e8..ee6f3a71 100644 --- a/test/state/state.go +++ b/test/state/state.go @@ -414,9 +414,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -649,11 +646,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -677,7 +669,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -731,27 +723,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -821,7 +801,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1116,7 +1096,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1159,7 +1138,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1304,7 +1282,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/stateclone/stateclone.go b/test/stateclone/stateclone.go index cea5e3b6..53380c3e 100644 --- a/test/stateclone/stateclone.go +++ b/test/stateclone/stateclone.go @@ -504,9 +504,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -739,11 +736,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -767,7 +759,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -821,27 +813,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -911,7 +891,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1206,7 +1186,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1249,7 +1228,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1394,7 +1372,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/statereadonly/statereadonly.go b/test/statereadonly/statereadonly.go index 38cd8f62..92455337 100644 --- a/test/statereadonly/statereadonly.go +++ b/test/statereadonly/statereadonly.go @@ -485,9 +485,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -720,11 +717,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -748,7 +740,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -802,27 +794,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -892,7 +872,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1187,7 +1167,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1230,7 +1209,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1375,7 +1353,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/optimized/staterestore.go b/test/staterestore/optimized/staterestore.go index 8909cedc..4adc27c7 100644 --- a/test/staterestore/optimized/staterestore.go +++ b/test/staterestore/optimized/staterestore.go @@ -860,9 +860,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1095,11 +1092,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1117,7 +1109,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1171,27 +1163,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1236,7 +1216,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1456,7 +1436,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1499,7 +1478,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } diff --git a/test/staterestore/standard/staterestore.go b/test/staterestore/standard/staterestore.go index 77dd3825..79c8761c 100644 --- a/test/staterestore/standard/staterestore.go +++ b/test/staterestore/standard/staterestore.go @@ -584,9 +584,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -819,11 +816,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -847,7 +839,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -901,27 +893,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -991,7 +971,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1286,7 +1266,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1329,7 +1308,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1474,7 +1452,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/staterestore.go b/test/staterestore/staterestore.go index 77dd3825..79c8761c 100644 --- a/test/staterestore/staterestore.go +++ b/test/staterestore/staterestore.go @@ -584,9 +584,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -819,11 +816,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -847,7 +839,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -901,27 +893,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -991,7 +971,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1286,7 +1266,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1329,7 +1308,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1474,7 +1452,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/thrownrecover/thrownrecover.go b/test/thrownrecover/thrownrecover.go index f0cafa8a..ed6914bb 100644 --- a/test/thrownrecover/thrownrecover.go +++ b/test/thrownrecover/thrownrecover.go @@ -1366,9 +1366,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -1601,11 +1598,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -1629,7 +1621,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -1683,27 +1675,15 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, rule) } func (p *parser) popRule() { p.rstack = p.rstack[:len(p.rstack)-1] } -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1] } // push a recovery expression with its labels to the recoveryStack @@ -1773,7 +1753,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.rstack[len(p.rstack)-1].rule + rule := p.getRule() if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -2068,7 +2048,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -2111,7 +2090,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -2256,7 +2234,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].rule.name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) From 32b14c87f52856663fbc28e219c0588d752db571 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Mon, 26 Jun 2023 09:44:17 +0300 Subject: [PATCH 06/14] Add docs --- doc.go | 23 +++++++++++++++++++++++ main.go | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc.go b/doc.go index bfa9acb8..1e1b285a 100644 --- a/doc.go +++ b/doc.go @@ -89,6 +89,12 @@ The following options can be specified: necessary if the -optimize-parser flag is set, as some rules may be optimized out of the resulting parser. + -support-left-recursion : boolean, (EXPERIMENTAL FEATURE) if set, add support + for left recursion rules, including those with indirect recursion + (default: false). + E.g.: + expr = expr '*' term / expr '+' term + If the code blocks in the grammar (see below, section "Code block") are golint- and go vet-compliant, then the resulting generated code will also be golint- and go vet-compliant. @@ -382,6 +388,23 @@ of pigeon and should not be used nor modified. Those keys are treated as internal implementation details and therefore there are no guarantees given in regards of API stability. +Left recursion + +With options -support-left-recursion pigeon supports left recursion. E.g.: + expr = expr '*' term +Supports indirect recursion: + A = B / D + B = A / C +Supports priorities: + expr = expr '*' term / expr '+' term +The implementation is based on the [Left-recursive PEG Grammars][9] article that +links to [Left Recursion in Parsing Expression Grammars][10] and +[Packrat Parsers Can Support Left Recursion][11] papers. +References: + [9]: https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1 + [10]: https://arxiv.org/pdf/1207.0443.pdf + [11]: http://web.cs.ucla.edu/~todd/research/pepm08.pdf + Failure labels, throw and recover pigeon supports an extension of the classical PEG syntax called failure labels, diff --git a/main.go b/main.go index c5580450..582c1636 100644 --- a/main.go +++ b/main.go @@ -48,10 +48,10 @@ func main() { outputFlag = fs.String("o", "", "output file, defaults to stdout") optimizeBasicLatinFlag = fs.Bool("optimize-basic-latin", false, "generate optimized parser for Unicode Basic Latin character sets") optimizeGrammar = fs.Bool("optimize-grammar", false, "optimize the given grammar (EXPERIMENTAL FEATURE)") - supportLeftRecursion = fs.Bool("support-left-recursion", false, "add support left recursion") optimizeParserFlag = fs.Bool("optimize-parser", false, "generate optimized parser without Debug and Memoize options") recvrNmFlag = fs.String("receiver-name", "c", "receiver name for the generated methods") noBuildFlag = fs.Bool("x", false, "do not build, only parse") + supportLeftRecursion = fs.Bool("support-left-recursion", false, "add support left recursion (EXPERIMENTAL FEATURE)") altEntrypointsFlag ruleNamesFlag ) @@ -211,6 +211,8 @@ the generated code is written to this file instead. comma-separated list of rule names that may be used as alternate entrypoints for the parser, in addition to the first rule in the grammar. + -support-left-recursion + add support left recursion (EXPERIMENTAL FEATURE) See https://godoc.org/github.com/mna/pigeon for more information. ` From 3f1e0796da67be760073d13f158ca68b1b2c1868 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Wed, 28 Jun 2023 15:46:10 +0300 Subject: [PATCH 07/14] Fix comments Replace map[string]bool with map[string]struct{} Add msgs in tests Correct typos Add testify library license Fix flaky test --- Makefile | 9 +- THIRD-PARTY-NOTICES | 25 + ast/ast.go | 87 +- builder/builder.go | 6 +- builder/generated_static_code.go | 19 +- builder/left_recursion.go | 30 +- builder/left_recursion_test.go | 36 +- builder/scc.go | 55 +- builder/scc_test.go | 302 +-- builder/static_code.go | 19 +- doc.go | 4 +- test/issue_70b/issue_70b.go | 19 +- test/left_recursion/left_recursion.go | 19 +- .../optimized/left_recursion.go | 1424 ++++++++++++++ .../{ => optimized}/left_recursion_test.go | 0 .../left_recursion/standard/left_recursion.go | 1749 +++++++++++++++++ .../standard/left_recursion_test.go | 26 + test/max_expr_cnt/maxexpr.peg | 2 +- testutils/testutils.go | 128 ++ 19 files changed, 3609 insertions(+), 350 deletions(-) create mode 100644 THIRD-PARTY-NOTICES create mode 100644 test/left_recursion/optimized/left_recursion.go rename test/left_recursion/{ => optimized}/left_recursion_test.go (100%) create mode 100644 test/left_recursion/standard/left_recursion.go create mode 100644 test/left_recursion/standard/left_recursion_test.go create mode 100644 testutils/testutils.go diff --git a/Makefile b/Makefile index d2c04fad..161ce897 100644 --- a/Makefile +++ b/Makefile @@ -172,9 +172,16 @@ $(TEST_DIR)/issue_70b/issue_70b.go: $(TEST_DIR)/issue_70b/issue_70b.peg $(BINDIR $(TEST_DIR)/issue_80/issue_80.go: $(TEST_DIR)/issue_80/issue_80.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ -$(TEST_DIR)/left_recursion/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon +$(TEST_DIR)/left_recursion/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(TEST_DIR)/left_recursion/standard/left_recursion.go $(TEST_DIR)/left_recursion/optimized/left_recursion.go $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ +$(TEST_DIR)/left_recursion/standard/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ + +$(TEST_DIR)/left_recursion/optimized/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -optimize-parser -support-left-recursion $< > $@ + + lint: golint ./... go vet ./... diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..14da9e13 --- /dev/null +++ b/THIRD-PARTY-NOTICES @@ -0,0 +1,25 @@ +---------------------------------------------------------------------- +License notice for github.com/stretchr/testify +---------------------------------------------------------------------- + +MIT License + +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ast/ast.go b/ast/ast.go index 19831898..13d34b7c 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -69,7 +69,7 @@ func (g *Grammar) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (g *Grammar) InitialNames() map[string]bool { +func (g *Grammar) InitialNames() map[string]struct{} { panic("InitialNames should not be called on the Grammar") } @@ -113,6 +113,7 @@ func (r *Rule) NullableVisit(rules map[string]*Rule) bool { } r.Visited = true r.Nullable = r.Expr.NullableVisit(rules) + r.Visited = false return r.Nullable } @@ -122,7 +123,7 @@ func (r *Rule) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (r *Rule) InitialNames() map[string]bool { +func (r *Rule) InitialNames() map[string]struct{} { return r.Expr.InitialNames() } @@ -133,7 +134,7 @@ type Expression interface { // for work with left recursion NullableVisit(rules map[string]*Rule) bool IsNullable() bool - InitialNames() map[string]bool + InitialNames() map[string]struct{} } // ChoiceExpr is an ordered sequence of expressions. The parser tries to @@ -186,11 +187,11 @@ func (c *ChoiceExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (c *ChoiceExpr) InitialNames() map[string]bool { - names := make(map[string]bool) +func (c *ChoiceExpr) InitialNames() map[string]struct{} { + names := make(map[string]struct{}) for _, alt := range c.Alternatives { for name := range alt.InitialNames() { - names[name] = true + names[name] = struct{}{} } } return names @@ -246,13 +247,13 @@ func (r *RecoveryExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (r *RecoveryExpr) InitialNames() map[string]bool { - names := make(map[string]bool) +func (r *RecoveryExpr) InitialNames() map[string]struct{} { + names := make(map[string]struct{}) for name := range r.Expr.InitialNames() { - names[name] = true + names[name] = struct{}{} } for name := range r.RecoverExpr.InitialNames() { - names[name] = true + names[name] = struct{}{} } return names } @@ -295,10 +296,10 @@ func (a *ActionExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (a *ActionExpr) InitialNames() map[string]bool { - names := make(map[string]bool) +func (a *ActionExpr) InitialNames() map[string]struct{} { + names := make(map[string]struct{}) for name := range a.Expr.InitialNames() { - names[name] = true + names[name] = struct{}{} } return names } @@ -336,8 +337,8 @@ func (t *ThrowExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (t *ThrowExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (t *ThrowExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // SeqExpr is an ordered sequence of expressions, all of which must match @@ -389,11 +390,11 @@ func (s *SeqExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (s *SeqExpr) InitialNames() map[string]bool { - names := make(map[string]bool) +func (s *SeqExpr) InitialNames() map[string]struct{} { + names := make(map[string]struct{}) for _, item := range s.Exprs { for name := range item.InitialNames() { - names[name] = true + names[name] = struct{}{} } if !item.IsNullable() { break @@ -437,7 +438,7 @@ func (l *LabeledExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (l *LabeledExpr) InitialNames() map[string]bool { +func (l *LabeledExpr) InitialNames() map[string]struct{} { return l.Expr.InitialNames() } @@ -474,8 +475,8 @@ func (a *AndExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (a *AndExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (a *AndExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // NotExpr is a zero-length matcher that is considered a match if the @@ -511,8 +512,8 @@ func (n *NotExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (n *NotExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (n *NotExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // ZeroOrOneExpr is an expression that can be matched zero or one time. @@ -548,7 +549,7 @@ func (z *ZeroOrOneExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (z *ZeroOrOneExpr) InitialNames() map[string]bool { +func (z *ZeroOrOneExpr) InitialNames() map[string]struct{} { return z.Expr.InitialNames() } @@ -585,7 +586,7 @@ func (z *ZeroOrMoreExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (z *ZeroOrMoreExpr) InitialNames() map[string]bool { +func (z *ZeroOrMoreExpr) InitialNames() map[string]struct{} { return z.Expr.InitialNames() } @@ -622,7 +623,7 @@ func (o *OneOrMoreExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (o *OneOrMoreExpr) InitialNames() map[string]bool { +func (o *OneOrMoreExpr) InitialNames() map[string]struct{} { return o.Expr.InitialNames() } @@ -668,8 +669,8 @@ func (r *RuleRefExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (r *RuleRefExpr) InitialNames() map[string]bool { - return map[string]bool{r.Name.Val: true} +func (r *RuleRefExpr) InitialNames() map[string]struct{} { + return map[string]struct{}{r.Name.Val: {}} } // StateCodeExpr is an expression which can modify the internal state of the parser. @@ -706,8 +707,8 @@ func (s *StateCodeExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (s *StateCodeExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (s *StateCodeExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // AndCodeExpr is a zero-length matcher that is considered a match if the @@ -745,8 +746,8 @@ func (a *AndCodeExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (a *AndCodeExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (a *AndCodeExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // NotCodeExpr is a zero-length matcher that is considered a match if the @@ -784,8 +785,8 @@ func (n *NotCodeExpr) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (n *NotCodeExpr) InitialNames() map[string]bool { - return make(map[string]bool) +func (n *NotCodeExpr) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // LitMatcher is a string literal matcher. The value to match may be a @@ -824,8 +825,8 @@ func (l *LitMatcher) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (l *LitMatcher) InitialNames() map[string]bool { - return make(map[string]bool) +func (l *LitMatcher) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // CharClassMatcher is a character class matcher. The value to match must @@ -976,8 +977,8 @@ func (c *CharClassMatcher) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (c *CharClassMatcher) InitialNames() map[string]bool { - return make(map[string]bool) +func (c *CharClassMatcher) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // AnyMatcher is a matcher that matches any character except end-of-file. @@ -1012,8 +1013,8 @@ func (a *AnyMatcher) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (a *AnyMatcher) InitialNames() map[string]bool { - return make(map[string]bool) +func (a *AnyMatcher) InitialNames() map[string]struct{} { + return make(map[string]struct{}) } // CodeBlock represents a code block. @@ -1048,7 +1049,7 @@ func (c *CodeBlock) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (c *CodeBlock) InitialNames() map[string]bool { +func (c *CodeBlock) InitialNames() map[string]struct{} { panic("InitialNames should not be called on the CodeBlock") } @@ -1084,7 +1085,7 @@ func (i *Identifier) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (i *Identifier) InitialNames() map[string]bool { +func (i *Identifier) InitialNames() map[string]struct{} { panic("InitialNames should not be called on the Identifier") } @@ -1120,7 +1121,7 @@ func (s *StringLit) IsNullable() bool { } // InitialNames returns names of nodes with which an expression can begin. -func (s *StringLit) InitialNames() map[string]bool { +func (s *StringLit) InitialNames() map[string]struct{} { panic("InitialNames should not be called on the StringLit") } diff --git a/builder/builder.go b/builder/builder.go index f93d2489..4d5b7d38 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -81,7 +81,7 @@ func Optimize(optimize bool) Option { // If supportLeftRecursion is true, LeftRecursion code is added to the resulting parser. func SupportLeftRecursion(support bool) Option { return func(b *builder) Option { - prev := b.optimize + prev := b.supportLeftRecursion b.supportLeftRecursion = support return SupportLeftRecursion(prev) } @@ -147,10 +147,10 @@ func (b *builder) setOptions(opts []Option) { func (b *builder) buildParser(grammar *ast.Grammar) error { haveLeftRecursion, err := PrepareGrammar(grammar) if err != nil { - return fmt.Errorf("uncorrect gramma: %w", err) + return fmt.Errorf("incorrect grammar: %w", err) } if !b.supportLeftRecursion && haveLeftRecursion { - return fmt.Errorf("uncorrect gramma: %w", ErrHaveLeftRecirsion) + return fmt.Errorf("incorrect grammar: %w", ErrHaveLeftRecursion) } b.haveLeftRecursion = haveLeftRecursion diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index b32c714c..6b067457 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -588,7 +588,7 @@ func (p *parser) popV() { func (p *parser) pushRule(rule *rule) { // ==template== {{ if .LeftRecursion }} - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) // {{ else }} p.rstack = append(p.rstack, rule) // {{ end }} ==template== @@ -612,7 +612,8 @@ func (p *parser) pushExpr(expr any) { return } p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) + p.rstack[len(p.rstack)-1].estack, expr, + ) } func (p *parser) popExpr() { @@ -620,7 +621,8 @@ func (p *parser) popExpr() { return } p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] + p.rstack[len(p.rstack)-1].estack, + )-1] } // {{ end }} ==template== @@ -935,15 +937,12 @@ func listJoin(list []string, sep string, lastSep string) string { // ==template== {{ if .LeftRecursion }} -func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { var currentEStack []any var prevEStack []any - for i := 1; i <= len(p.rstack); i++ { + for i := 1; i <= len(p.rstack)/2; i++ { indexCurrent := len(p.rstack) - i indexPrev := len(p.rstack) - i*2 - if indexPrev < 0 { - continue - } if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { currentEStack = p.rstack[indexCurrent].estack prevEStack = p.rstack[indexPrev].estack @@ -998,8 +997,8 @@ func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoices := p.checkPrevChoice(rule) - if haveChoices && !checkPriority { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { return nil, false } p.restore(result.end) diff --git a/builder/left_recursion.go b/builder/left_recursion.go index 6d668656..d669d4a9 100644 --- a/builder/left_recursion.go +++ b/builder/left_recursion.go @@ -11,8 +11,8 @@ var ( // ErrNoLeader is no leader error. ErrNoLeader = errors.New( "SCC has no leadership candidate (no element is included in all cycles)") - // ErrHaveLeftRecirsion is recursion error. - ErrHaveLeftRecirsion = errors.New("have left recursion") + // ErrHaveLeftRecursion is recursion error. + ErrHaveLeftRecursion = errors.New("grammar contains left recursion") ) // PrepareGrammar evaluates parameters associated with left recursion. @@ -38,18 +38,22 @@ func ComputeNullables(rules map[string]*ast.Rule) { } func findLeader( - graph map[string]map[string]bool, scc map[string]bool, + graph map[string]map[string]struct{}, scc map[string]struct{}, ) (string, error) { // Try to find a leader such that all cycles go through it. - leaders := make(map[string]bool, len(scc)) + leaders := make(map[string]struct{}, len(scc)) for k := range scc { - leaders[k] = true + leaders[k] = struct{}{} } for start := range scc { - for _, cycle := range FindCyclesInSCC(graph, scc, start) { - mapCycle := map[string]bool{} + cycles, err := FindCyclesInSCC(graph, scc, start) + if err != nil { + return "", fmt.Errorf("error find cycles: %w", err) + } + for _, cycle := range cycles { + mapCycle := make(map[string]struct{}, len(cycle)) for _, k := range cycle { - mapCycle[k] = true + mapCycle[k] = struct{}{} } for k := range scc { if _, okCycle := mapCycle[k]; !okCycle { @@ -109,19 +113,19 @@ func ComputeLeftRecursives(rules map[string]*ast.Rule) (bool, error) { // MakeFirstGraph compute the graph of left-invocations. // There's an edge from A to B if A may invoke B at its initial position. // Note that this requires the nullable flags to have been computed. -func MakeFirstGraph(rules map[string]*ast.Rule) map[string]map[string]bool { - graph := make(map[string]map[string]bool) - vertices := make(map[string]bool) +func MakeFirstGraph(rules map[string]*ast.Rule) map[string]map[string]struct{} { + graph := make(map[string]map[string]struct{}) + vertices := make(map[string]struct{}) for rulename, rule := range rules { names := rule.InitialNames() graph[rulename] = names for name := range names { - vertices[name] = true + vertices[name] = struct{}{} } } for vertex := range vertices { if _, ok := graph[vertex]; !ok { - graph[vertex] = make(map[string]bool) + graph[vertex] = make(map[string]struct{}) } } return graph diff --git a/builder/left_recursion_test.go b/builder/left_recursion_test.go index 99ac1bc7..96a5b09d 100644 --- a/builder/left_recursion_test.go +++ b/builder/left_recursion_test.go @@ -39,22 +39,22 @@ func TestLeftRecursive(t *testing.T) { mapRules[rule.Name.Val] = rule } if mapRules["start"].LeftRecursive { - t.Fail() + t.Error("Rule 'start' does not contain left recursion") } if !mapRules["expr"].LeftRecursive { - t.Fail() + t.Error("Rule 'expr' contains left recursion") } if mapRules["term"].LeftRecursive { - t.Fail() + t.Error("Rule 'term' does not contain left recursion") } if mapRules["foo"].LeftRecursive { - t.Fail() + t.Error("Rule 'foo' does not contain left recursion") } if mapRules["bar"].LeftRecursive { - t.Fail() + t.Error("Rule 'bar' does not contain left recursion") } if mapRules["baz"].LeftRecursive { - t.Fail() + t.Error("Rule 'baz' does not contain left recursion") } } @@ -82,10 +82,10 @@ func TestNullable(t *testing.T) { mapRules[rule.Name.Val] = rule } if mapRules["start"].Nullable { - t.Fail() + t.Error("Rule 'start' is not nullable") } if !mapRules["sign"].Nullable { - t.Fail() + t.Error("Rule 'sign' is nullable") } } @@ -113,16 +113,16 @@ func TestAdvancedLeftRecursive(t *testing.T) { mapRules[rule.Name.Val] = rule } if mapRules["start"].Nullable { - t.Fail() + t.Error("Rule 'start' is not Nullable") } if !mapRules["sign"].Nullable { - t.Fail() + t.Error("Rule 'sign' is Nullable") } if !mapRules["start"].LeftRecursive { - t.Fail() + t.Error("Rule 'start' does not contain left recursion") } if mapRules["sign"].LeftRecursive { - t.Fail() + t.Error("Rule 'sign' contains left recursion") } } @@ -151,13 +151,13 @@ func TestMutuallyLeftRecursive(t *testing.T) { mapRules[rule.Name.Val] = rule } if mapRules["start"].LeftRecursive { - t.Fail() + t.Error("Rule 'start' does not contain left recursion") } if !mapRules["foo"].LeftRecursive { - t.Fail() + t.Error("Rule 'foo' contains left recursion") } if !mapRules["bar"].LeftRecursive { - t.Fail() + t.Error("Rule 'bar' contains left recursion") } } @@ -186,13 +186,13 @@ func TestNastyMutuallyLeftRecursive(t *testing.T) { mapRules[rule.Name.Val] = rule } if mapRules["start"].LeftRecursive { - t.Fail() + t.Error("Rule 'start' does not contain left recursion") } if !mapRules["target"].LeftRecursive { - t.Fail() + t.Error("Rule 'target' contains left recursion") } if !mapRules["maybe"].LeftRecursive { - t.Fail() + t.Error("Rule 'maybe' contains left recursion") } } diff --git a/builder/scc.go b/builder/scc.go index 111a8ce6..6e8381c2 100644 --- a/builder/scc.go +++ b/builder/scc.go @@ -1,6 +1,12 @@ package builder -import "fmt" +import ( + "errors" + "fmt" +) + +// ErrInvalidParameters is parameters error. +var ErrInvalidParameters = errors.New("invalid parameters passed to function") func min(a1 int, a2 int) int { if a1 <= a2 { @@ -12,23 +18,23 @@ func min(a1 int, a2 int) int { // StronglyConnectedComponents compute strongly сonnected сomponents of a graph. // Tarjan's strongly connected components algorithm. func StronglyConnectedComponents( - vertices []string, edges map[string]map[string]bool, -) []map[string]bool { + vertices []string, edges map[string]map[string]struct{}, +) []map[string]struct{} { // Tarjan's strongly connected components algorithm var ( - identified = map[string]bool{} + identified = map[string]struct{}{} stack = []string{} index = map[string]int{} lowlink = map[string]int{} - dfs func(v string) []map[string]bool + dfs func(v string) []map[string]struct{} ) - dfs = func(vertex string) []map[string]bool { + dfs = func(vertex string) []map[string]struct{} { index[vertex] = len(stack) stack = append(stack, vertex) lowlink[vertex] = index[vertex] - sccs := []map[string]bool{} + sccs := []map[string]struct{}{} for w := range edges[vertex] { if _, ok := index[w]; !ok { sccs = append(sccs, dfs(w)...) @@ -39,20 +45,20 @@ func StronglyConnectedComponents( } if lowlink[vertex] == index[vertex] { - scc := map[string]bool{} + scc := map[string]struct{}{} for _, v := range stack[index[vertex]:] { - scc[v] = true + scc[v] = struct{}{} } stack = stack[:index[vertex]] for v := range scc { - identified[v] = true + identified[v] = struct{}{} } sccs = append(sccs, scc) } return sccs } - sccs := []map[string]bool{} + sccs := []map[string]struct{}{} for _, v := range vertices { if _, ok := index[v]; !ok { sccs = append(sccs, dfs(v)...) @@ -71,19 +77,19 @@ func contains(s []string, e string) bool { } func reduceGraph( - graph map[string]map[string]bool, scc map[string]bool, -) map[string]map[string]bool { - reduceGraph := map[string]map[string]bool{} + graph map[string]map[string]struct{}, scc map[string]struct{}, +) map[string]map[string]struct{} { + reduceGraph := map[string]map[string]struct{}{} for src, dsts := range graph { if _, ok := scc[src]; !ok { continue } - reduceGraph[src] = map[string]bool{} + reduceGraph[src] = map[string]struct{}{} for dst := range dsts { if _, ok := scc[dst]; !ok { continue } - reduceGraph[src][dst] = true + reduceGraph[src][dst] = struct{}{} } } return reduceGraph @@ -96,11 +102,12 @@ func reduceGraph( // 'B', 'C', 'B'] means there's a path from A to B and there's a // cycle from B to C and back. func FindCyclesInSCC( - graph map[string]map[string]bool, scc map[string]bool, start string, -) [][]string { + graph map[string]map[string]struct{}, scc map[string]struct{}, start string, +) ([][]string, error) { // Basic input checks. if _, ok := scc[start]; !ok { - panic(fmt.Sprintf("scc %v have not %v", scc, start)) + return nil, fmt.Errorf( + "%w: scc %v does not contain %q", ErrInvalidParameters, scc, start) } extravertices := []string{} for k := range scc { @@ -109,13 +116,17 @@ func FindCyclesInSCC( } } if len(extravertices) != 0 { - panic(fmt.Sprintf("graph have not scc. %v", extravertices)) + return nil, fmt.Errorf( + "%w: graph does not contain scc. %v", + ErrInvalidParameters, extravertices) } // Reduce the graph to nodes in the SCC. graph = reduceGraph(graph, scc) if _, ok := graph[start]; !ok { - panic(fmt.Sprintf("graph %v have not %v", graph, start)) + return nil, fmt.Errorf( + "%w: graph %v does not contain %q", + ErrInvalidParameters, graph, start) } // Recursive helper that yields cycles. @@ -136,5 +147,5 @@ func FindCyclesInSCC( return ret } - return dfs(start, []string{}) + return dfs(start, []string{}), nil } diff --git a/builder/scc_test.go b/builder/scc_test.go index 9af37818..1b7d4661 100644 --- a/builder/scc_test.go +++ b/builder/scc_test.go @@ -1,214 +1,97 @@ package builder_test import ( - "bytes" - "reflect" "testing" "github.com/mna/pigeon/builder" + "github.com/mna/pigeon/testutils" ) -// isEmpty gets whether the specified object is considered empty or not. -func isEmpty(object interface{}) bool { - // get nil case out of the way - if object == nil { - return true - } - - objValue := reflect.ValueOf(object) - - switch objValue.Kind() { - // collection types are empty when they have no element - case reflect.Chan, reflect.Map, reflect.Slice: - return objValue.Len() == 0 - // pointers are empty if nil or if the value they point to is empty - case reflect.Ptr: - if objValue.IsNil() { - return true - } - deref := objValue.Elem().Interface() - return isEmpty(deref) - // for all other types, compare against the zero value - // array types are empty when they match their zero-initialized state - default: - zero := reflect.Zero(objValue.Type()) - return reflect.DeepEqual(object, zero.Interface()) - } -} - -// isList checks that the provided value is array or slice. -func isList(list interface{}) (ok bool) { - kind := reflect.TypeOf(list).Kind() - return kind == reflect.Array || kind == reflect.Slice -} - -// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. -// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and -// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. -func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { - aValue := reflect.ValueOf(listA) - bValue := reflect.ValueOf(listB) - - aLen := aValue.Len() - bLen := bValue.Len() - - // Mark indexes in bValue that we already used - visited := make([]bool, bLen) - for i := 0; i < aLen; i++ { - element := aValue.Index(i).Interface() - found := false - for j := 0; j < bLen; j++ { - if visited[j] { - continue - } - if ObjectsAreEqual(bValue.Index(j).Interface(), element) { - visited[j] = true - found = true - break - } - } - if !found { - extraA = append(extraA, element) - } - } - - for j := 0; j < bLen; j++ { - if visited[j] { - continue - } - extraB = append(extraB, bValue.Index(j).Interface()) - } - - return -} - -// ObjectsAreEqual determines if two objects are considered equal. -// -// This function does no assertion of any kind. -func ObjectsAreEqual(expected, actual interface{}) bool { - if expected == nil || actual == nil { - return expected == actual - } - - exp, ok := expected.([]byte) - if !ok { - return reflect.DeepEqual(expected, actual) - } - - act, ok := actual.([]byte) - if !ok { - return false - } - if exp == nil || act == nil { - return exp == nil && act == nil - } - return bytes.Equal(exp, act) -} - -// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified -// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, -// the number of appearances of each of them in both lists should match. -// -// ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]). -func ElementsMatch(listA interface{}, listB interface{}) bool { - if isEmpty(listA) && isEmpty(listB) { - return true - } - - if !isList(listA) || !isList(listB) { - return false - } - - extraA, extraB := diffLists(listA, listB) - - return len(extraA) == 0 && len(extraB) == 0 -} - func TestStronglyConnectedComponents(t *testing.T) { //nolint:funlen t.Parallel() type want struct { - sccs []map[string]bool + sccs []map[string]struct{} } tests := []struct { name string - graph map[string]map[string]bool + graph map[string]map[string]struct{} want want }{ { name: "Simple", - graph: map[string]map[string]bool{ - "1": {"2": true}, - "2": {"1": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, + "2": {"1": {}}, }, - want: want{sccs: []map[string]bool{ - {"2": true, "1": true}, + want: want{sccs: []map[string]struct{}{ + {"2": {}, "1": {}}, }}, }, { name: "Without scc", - graph: map[string]map[string]bool{ - "1": {"2": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, }, - want: want{sccs: []map[string]bool{ - {"2": true}, - {"1": true}, + want: want{sccs: []map[string]struct{}{ + {"2": {}}, + {"1": {}}, }}, }, { name: "One element", - graph: map[string]map[string]bool{ + graph: map[string]map[string]struct{}{ "1": {}, }, - want: want{sccs: []map[string]bool{ - {"1": true}, + want: want{sccs: []map[string]struct{}{ + {"1": {}}, }}, }, { name: "One element with loop", - graph: map[string]map[string]bool{ - "1": {"1": true}, + graph: map[string]map[string]struct{}{ + "1": {"1": {}}, }, - want: want{sccs: []map[string]bool{ - {"1": true}, + want: want{sccs: []map[string]struct{}{ + {"1": {}}, }}, }, { name: "Wiki 1", - graph: map[string]map[string]bool{ - "1": {"2": true}, - "2": {"3": true}, - "3": {"1": true}, - "4": {"2": true, "3": true, "6": true}, - "5": {"3": true, "7": true}, - "6": {"4": true, "5": true}, - "7": {"5": true}, - "8": {"6": true, "7": true, "8": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, + "2": {"3": {}}, + "3": {"1": {}}, + "4": {"2": {}, "3": {}, "6": {}}, + "5": {"3": {}, "7": {}}, + "6": {"4": {}, "5": {}}, + "7": {"5": {}}, + "8": {"6": {}, "7": {}, "8": {}}, }, - want: want{sccs: []map[string]bool{ - {"2": true, "3": true, "1": true}, - {"5": true, "7": true}, - {"4": true, "6": true}, - {"8": true}, + want: want{sccs: []map[string]struct{}{ + {"2": {}, "3": {}, "1": {}}, + {"5": {}, "7": {}}, + {"4": {}, "6": {}}, + {"8": {}}, }}, }, { name: "Wiki 2", - graph: map[string]map[string]bool{ - "1": {"2": true, "6": true}, - "2": {"6": true, "4": true}, - "3": {"9": true, "4": true, "8": true}, - "4": {"1": true, "7": true}, - "5": {"9": true, "8": true}, - "6": {"1": true, "4": true, "7": true}, - "7": {"1": true}, - "8": {"5": true, "3": true}, - "9": {"8": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}, "6": {}}, + "2": {"6": {}, "4": {}}, + "3": {"9": {}, "4": {}, "8": {}}, + "4": {"1": {}, "7": {}}, + "5": {"9": {}, "8": {}}, + "6": {"1": {}, "4": {}, "7": {}}, + "7": {"1": {}}, + "8": {"5": {}, "3": {}}, + "9": {"8": {}}, }, - want: want{sccs: []map[string]bool{ - {"1": true, "2": true, "4": true, "6": true, "7": true}, - {"3": true, "5": true, "9": true, "8": true}, + want: want{sccs: []map[string]struct{}{ + {"1": {}, "2": {}, "4": {}, "6": {}, "7": {}}, + {"3": {}, "5": {}, "9": {}, "8": {}}, }}, }, } @@ -222,8 +105,8 @@ func TestStronglyConnectedComponents(t *testing.T) { //nolint:funlen vertices = append(vertices, k) } sccs := builder.StronglyConnectedComponents(vertices, testCase.graph) - if !ElementsMatch(sccs, testCase.want.sccs) { - t.FailNow() + if !testutils.ElementsMatch(sccs, testCase.want.sccs) { + t.Fatalf("Result %v, expected %v", sccs, testCase.want.sccs) } }) } @@ -238,58 +121,58 @@ func TestFindCyclesInSCC(t *testing.T) { //nolint:funlen tests := []struct { name string - graph map[string]map[string]bool - scc map[string]bool + graph map[string]map[string]struct{} + scc map[string]struct{} start string want want }{ { name: "Wiki 1 1", - graph: map[string]map[string]bool{ - "1": {"2": true}, - "2": {"3": true}, - "3": {"1": true}, - "4": {"2": true, "3": true, "6": true}, - "5": {"3": true, "7": true}, - "6": {"4": true, "5": true}, - "7": {"5": true}, - "8": {"6": true, "7": true, "8": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, + "2": {"3": {}}, + "3": {"1": {}}, + "4": {"2": {}, "3": {}, "6": {}}, + "5": {"3": {}, "7": {}}, + "6": {"4": {}, "5": {}}, + "7": {"5": {}}, + "8": {"6": {}, "7": {}, "8": {}}, }, - scc: map[string]bool{"2": true, "3": true, "1": true}, + scc: map[string]struct{}{"2": {}, "3": {}, "1": {}}, start: "3", want: want{paths: [][]string{{"3", "1", "2", "3"}}}, }, { name: "Wiki 1 2", - graph: map[string]map[string]bool{ - "1": {"2": true}, - "2": {"3": true}, - "3": {"1": true}, - "4": {"2": true, "3": true, "6": true}, - "5": {"3": true, "7": true}, - "6": {"4": true, "5": true}, - "7": {"5": true}, - "8": {"6": true, "7": true, "8": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, + "2": {"3": {}}, + "3": {"1": {}}, + "4": {"2": {}, "3": {}, "6": {}}, + "5": {"3": {}, "7": {}}, + "6": {"4": {}, "5": {}}, + "7": {"5": {}}, + "8": {"6": {}, "7": {}, "8": {}}, }, - scc: map[string]bool{"5": true, "7": true}, + scc: map[string]struct{}{"5": {}, "7": {}}, start: "5", want: want{paths: [][]string{{"5", "7", "5"}}}, }, { name: "Wiki 2", - graph: map[string]map[string]bool{ - "1": {"2": true, "6": true}, - "2": {"6": true, "4": true}, - "3": {"9": true, "4": true, "8": true}, - "4": {"1": true, "7": true}, - "5": {"9": true, "8": true}, - "6": {"1": true, "4": true, "7": true}, - "7": {"1": true}, - "8": {"5": true, "3": true}, - "9": {"8": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}, "6": {}}, + "2": {"6": {}, "4": {}}, + "3": {"9": {}, "4": {}, "8": {}}, + "4": {"1": {}, "7": {}}, + "5": {"9": {}, "8": {}}, + "6": {"1": {}, "4": {}, "7": {}}, + "7": {"1": {}}, + "8": {"5": {}, "3": {}}, + "9": {"8": {}}, }, - scc: map[string]bool{ - "1": true, "2": true, "4": true, "6": true, "7": true, + scc: map[string]struct{}{ + "1": {}, "2": {}, "4": {}, "6": {}, "7": {}, }, start: "1", want: want{paths: [][]string{ @@ -307,13 +190,13 @@ func TestFindCyclesInSCC(t *testing.T) { //nolint:funlen }, { name: "loop in loop", - graph: map[string]map[string]bool{ - "1": {"2": true}, - "2": {"3": true}, - "3": {"1": true, "2": true}, + graph: map[string]map[string]struct{}{ + "1": {"2": {}}, + "2": {"3": {}}, + "3": {"1": {}, "2": {}}, }, - scc: map[string]bool{ - "1": true, "2": true, "3": true, + scc: map[string]struct{}{ + "1": {}, "2": {}, "3": {}, }, start: "1", want: want{paths: [][]string{ @@ -326,11 +209,14 @@ func TestFindCyclesInSCC(t *testing.T) { //nolint:funlen testCase := testCase t.Run(testCase.name, func(t *testing.T) { t.Parallel() - paths := builder.FindCyclesInSCC( + paths, err := builder.FindCyclesInSCC( testCase.graph, testCase.scc, testCase.start) - if !ElementsMatch(paths, testCase.want.paths) { + if err != nil { t.FailNow() } + if !testutils.ElementsMatch(paths, testCase.want.paths) { + t.Fatalf("Result %v, expected %v", paths, testCase.want.paths) + } }) } } diff --git a/builder/static_code.go b/builder/static_code.go index 9b54c32c..bbe8c588 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -606,7 +606,7 @@ func (p *parser) popV() { func (p *parser) pushRule(rule *rule) { // ==template== {{ if .LeftRecursion }} - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) // {{ else }} p.rstack = append(p.rstack, rule) // {{ end }} ==template== @@ -630,7 +630,8 @@ func (p *parser) pushExpr(expr any) { return } p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) + p.rstack[len(p.rstack)-1].estack, expr, + ) } func (p *parser) popExpr() { @@ -638,7 +639,8 @@ func (p *parser) popExpr() { return } p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] + p.rstack[len(p.rstack)-1].estack, + )-1] } // {{ end }} ==template== @@ -953,15 +955,12 @@ func listJoin(list []string, sep string, lastSep string) string { // ==template== {{ if .LeftRecursion }} -func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { var currentEStack []any var prevEStack []any - for i := 1; i <= len(p.rstack); i++ { + for i := 1; i <= len(p.rstack)/2; i++ { indexCurrent := len(p.rstack) - i indexPrev := len(p.rstack) - i*2 - if indexPrev < 0 { - continue - } if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { currentEStack = p.rstack[indexCurrent].estack prevEStack = p.rstack[indexPrev].estack @@ -1016,8 +1015,8 @@ func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoices := p.checkPrevChoice(rule) - if haveChoices && !checkPriority { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { return nil, false } p.restore(result.end) diff --git a/doc.go b/doc.go index 1e1b285a..360c1ca9 100644 --- a/doc.go +++ b/doc.go @@ -395,12 +395,14 @@ With options -support-left-recursion pigeon supports left recursion. E.g.: Supports indirect recursion: A = B / D B = A / C -Supports priorities: +Preserve support for ordered choices: expr = expr '*' term / expr '+' term The implementation is based on the [Left-recursive PEG Grammars][9] article that links to [Left Recursion in Parsing Expression Grammars][10] and [Packrat Parsers Can Support Left Recursion][11] papers. + References: + [9]: https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1 [10]: https://arxiv.org/pdf/1207.0443.pdf [11]: http://web.cs.ucla.edu/~todd/research/pepm08.pdf diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index d5af0fb1..47208a62 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -637,7 +637,7 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) } func (p *parser) popRule() { @@ -653,7 +653,8 @@ func (p *parser) pushExpr(expr any) { return } p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) + p.rstack[len(p.rstack)-1].estack, expr, + ) } func (p *parser) popExpr() { @@ -661,7 +662,8 @@ func (p *parser) popExpr() { return } p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] + p.rstack[len(p.rstack)-1].estack, + )-1] } // push a recovery expression with its labels to the recoveryStack @@ -954,15 +956,12 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { var currentEStack []any var prevEStack []any - for i := 1; i <= len(p.rstack); i++ { + for i := 1; i <= len(p.rstack)/2; i++ { indexCurrent := len(p.rstack) - i indexPrev := len(p.rstack) - i*2 - if indexPrev < 0 { - continue - } if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { currentEStack = p.rstack[indexCurrent].estack prevEStack = p.rstack[indexPrev].estack @@ -1017,8 +1016,8 @@ func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoices := p.checkPrevChoice(rule) - if haveChoices && !checkPriority { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { return nil, false } p.restore(result.end) diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/left_recursion.go index a9959d11..a3dffeae 100644 --- a/test/left_recursion/left_recursion.go +++ b/test/left_recursion/left_recursion.go @@ -818,7 +818,7 @@ func (p *parser) popV() { } func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule, []any{}}) + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) } func (p *parser) popRule() { @@ -834,7 +834,8 @@ func (p *parser) pushExpr(expr any) { return } p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr) + p.rstack[len(p.rstack)-1].estack, expr, + ) } func (p *parser) popExpr() { @@ -842,7 +843,8 @@ func (p *parser) popExpr() { return } p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack)-1] + p.rstack[len(p.rstack)-1].estack, + )-1] } // push a recovery expression with its labels to the recoveryStack @@ -1135,15 +1137,12 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { var currentEStack []any var prevEStack []any - for i := 1; i <= len(p.rstack); i++ { + for i := 1; i <= len(p.rstack)/2; i++ { indexCurrent := len(p.rstack) - i indexPrev := len(p.rstack) - i*2 - if indexPrev < 0 { - continue - } if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { currentEStack = p.rstack[indexCurrent].estack prevEStack = p.rstack[indexPrev].estack @@ -1198,8 +1197,8 @@ func (p *parser) checkPrevChoice(rule *rule) (bool, bool) { func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoices := p.checkPrevChoice(rule) - if haveChoices && !checkPriority { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { return nil, false } p.restore(result.end) diff --git a/test/left_recursion/optimized/left_recursion.go b/test/left_recursion/optimized/left_recursion.go new file mode 100644 index 00000000..599cf1b2 --- /dev/null +++ b/test/left_recursion/optimized/left_recursion.go @@ -0,0 +1,1424 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursion + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 5, col: 1, offset: 29}, + expr: &actionExpr{ + pos: position{line: 5, col: 9, offset: 37}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 5, col: 9, offset: 37}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 5, col: 9, offset: 37}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 5, col: 11, offset: 39}, + name: "expr", + }, + }, + ¬Expr{ + pos: position{line: 5, col: 16, offset: 44}, + expr: &anyMatcher{ + line: 5, col: 17, offset: 45, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "expr", + pos: position{line: 8, col: 1, offset: 66}, + expr: &choiceExpr{ + pos: position{line: 8, col: 8, offset: 73}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 8, col: 8, offset: 73}, + run: (*parser).callonexpr2, + expr: &seqExpr{ + pos: position{line: 8, col: 8, offset: 73}, + exprs: []any{ + &litMatcher{ + pos: position{line: 8, col: 8, offset: 73}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + &labeledExpr{ + pos: position{line: 8, col: 12, offset: 77}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 8, col: 14, offset: 79}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 11, col: 5, offset: 152}, + run: (*parser).callonexpr7, + expr: &seqExpr{ + pos: position{line: 11, col: 5, offset: 152}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 11, col: 5, offset: 152}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 11, col: 7, offset: 154}, + name: "expr", + }, + }, + &labeledExpr{ + pos: position{line: 11, col: 12, offset: 159}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 11, col: 16, offset: 163}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 11, col: 16, offset: 163}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 11, col: 22, offset: 169}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 11, col: 28, offset: 175}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 11, col: 33, offset: 180}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 11, col: 35, offset: 182}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 16, col: 5, offset: 321}, + run: (*parser).callonexpr18, + expr: &seqExpr{ + pos: position{line: 16, col: 5, offset: 321}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 16, col: 5, offset: 321}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 16, col: 7, offset: 323}, + name: "expr", + }, + }, + &labeledExpr{ + pos: position{line: 16, col: 12, offset: 328}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 16, col: 16, offset: 332}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 16, col: 16, offset: 332}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 16, col: 22, offset: 338}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 16, col: 27, offset: 343}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 16, col: 29, offset: 345}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 21, col: 5, offset: 483}, + run: (*parser).callonexpr28, + expr: &ruleRefExpr{ + pos: position{line: 21, col: 5, offset: 483}, + name: "term", + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 25, col: 1, offset: 524}, + expr: &oneOrMoreExpr{ + pos: position{line: 25, col: 8, offset: 531}, + expr: &charClassMatcher{ + pos: position{line: 25, col: 8, offset: 531}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onstart1(a any) (any, error) { + return a, nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1(stack["a"]) +} + +func (c *current) onexpr2(a any) (any, error) { + strA := a.(string) + return "(" + "-" + strA + ")", nil +} + +func (p *parser) callonexpr2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr2(stack["a"]) +} + +func (c *current) onexpr7(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonexpr7() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr18(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonexpr18() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr28() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonexpr28() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr28() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []ruleWithExpsStack + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1].rule +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr, + ) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack, + )-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.getRule() + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack)/2; i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { + return nil, false + } + p.restore(result.end) + return result.v, result.b + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + + for { + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if (!ok) || (endMark.offset <= lastResult.end.offset) { + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + if rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.pushRule(rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.popRule() + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + p.pushExpr(expr) + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + p.popExpr() + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + + val = actVal + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + pt := p.pt + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + return val, ok + } + } + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + pt := p.pt + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/optimized/left_recursion_test.go similarity index 100% rename from test/left_recursion/left_recursion_test.go rename to test/left_recursion/optimized/left_recursion_test.go diff --git a/test/left_recursion/standard/left_recursion.go b/test/left_recursion/standard/left_recursion.go new file mode 100644 index 00000000..a3dffeae --- /dev/null +++ b/test/left_recursion/standard/left_recursion.go @@ -0,0 +1,1749 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursion + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 5, col: 1, offset: 29}, + expr: &actionExpr{ + pos: position{line: 5, col: 9, offset: 37}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 5, col: 9, offset: 37}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 5, col: 9, offset: 37}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 5, col: 11, offset: 39}, + name: "expr", + }, + }, + ¬Expr{ + pos: position{line: 5, col: 16, offset: 44}, + expr: &anyMatcher{ + line: 5, col: 17, offset: 45, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "expr", + pos: position{line: 8, col: 1, offset: 66}, + expr: &choiceExpr{ + pos: position{line: 8, col: 8, offset: 73}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 8, col: 8, offset: 73}, + run: (*parser).callonexpr2, + expr: &seqExpr{ + pos: position{line: 8, col: 8, offset: 73}, + exprs: []any{ + &litMatcher{ + pos: position{line: 8, col: 8, offset: 73}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + &labeledExpr{ + pos: position{line: 8, col: 12, offset: 77}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 8, col: 14, offset: 79}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 11, col: 5, offset: 152}, + run: (*parser).callonexpr7, + expr: &seqExpr{ + pos: position{line: 11, col: 5, offset: 152}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 11, col: 5, offset: 152}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 11, col: 7, offset: 154}, + name: "expr", + }, + }, + &labeledExpr{ + pos: position{line: 11, col: 12, offset: 159}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 11, col: 16, offset: 163}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 11, col: 16, offset: 163}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 11, col: 22, offset: 169}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 11, col: 28, offset: 175}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 11, col: 33, offset: 180}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 11, col: 35, offset: 182}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 16, col: 5, offset: 321}, + run: (*parser).callonexpr18, + expr: &seqExpr{ + pos: position{line: 16, col: 5, offset: 321}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 16, col: 5, offset: 321}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 16, col: 7, offset: 323}, + name: "expr", + }, + }, + &labeledExpr{ + pos: position{line: 16, col: 12, offset: 328}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 16, col: 16, offset: 332}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 16, col: 16, offset: 332}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 16, col: 22, offset: 338}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 16, col: 27, offset: 343}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 16, col: 29, offset: 345}, + name: "expr", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 21, col: 5, offset: 483}, + run: (*parser).callonexpr28, + expr: &ruleRefExpr{ + pos: position{line: 21, col: 5, offset: 483}, + name: "term", + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 25, col: 1, offset: 524}, + expr: &oneOrMoreExpr{ + pos: position{line: 25, col: 8, offset: 531}, + expr: &charClassMatcher{ + pos: position{line: 25, col: 8, offset: 531}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onstart1(a any) (any, error) { + return a, nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1(stack["a"]) +} + +func (c *current) onexpr2(a any) (any, error) { + strA := a.(string) + return "(" + "-" + strA + ")", nil +} + +func (p *parser) callonexpr2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr2(stack["a"]) +} + +func (c *current) onexpr7(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonexpr7() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr18(a, op, b any) (any, error) { + strA := a.(string) + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} + +func (p *parser) callonexpr18() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr28() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonexpr28() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr28() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []ruleWithExpsStack + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +func (p *parser) pushRule(rule *rule) { + p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) +} + +func (p *parser) popRule() { + p.rstack = p.rstack[:len(p.rstack)-1] +} + +func (p *parser) getRule() *rule { + return p.rstack[len(p.rstack)-1].rule +} + +func (p *parser) pushExpr(expr any) { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = append( + p.rstack[len(p.rstack)-1].estack, expr, + ) +} + +func (p *parser) popExpr() { + if len(p.rstack) == 0 { + return + } + p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( + p.rstack[len(p.rstack)-1].estack, + )-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.getRule() + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { + var currentEStack []any + var prevEStack []any + for i := 1; i <= len(p.rstack)/2; i++ { + indexCurrent := len(p.rstack) - i + indexPrev := len(p.rstack) - i*2 + if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { + currentEStack = p.rstack[indexCurrent].estack + prevEStack = p.rstack[indexPrev].estack + break + } + } + if prevEStack == nil || currentEStack == nil { + return false, false + } + if len(prevEStack) != len(currentEStack) { + panic("Stacks are not equal(len)") + } + + for i := len(prevEStack) - 1; i >= 0; i-- { + currentCh, ok := currentEStack[i].(*choiceExpr) + if !ok { + continue + } + prevCh, ok := prevEStack[i].(*choiceExpr) + if !ok { + panic("Stacks are not equal(position choiceExpr)") + } + if currentCh != prevCh { + panic("Stacks are not equal(choiceExpr)") + } + currentAlt := -1 + for j, inExp := range currentCh.alternatives { + if inExp == currentEStack[i+1] { + currentAlt = j + break + } + } + if currentAlt == -1 { + panic("lost alternatives in choiceExpr") + } + prevAlt := -1 + for j, inExp := range prevCh.alternatives { + if inExp == prevEStack[i+1] { + prevAlt = j + break + } + } + if prevAlt == -1 { + panic("lost alternatives in choiceExpr") + } + return currentAlt < prevAlt, true + } + + return false, false +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + checkPriority, haveChoice := p.checkPrevChoice(rule) + if haveChoice && !checkPriority { + return nil, false + } + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) + } + if (!ok) || (endMark.offset <= lastResult.end.offset) { + p.restoreState(lastState) + break + } + lastResult = resultTuple{val, ok, endMark} + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.pushRule(rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.popRule() + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + p.pushExpr(expr) + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + p.popExpr() + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion/standard/left_recursion_test.go b/test/left_recursion/standard/left_recursion_test.go new file mode 100644 index 00000000..63412630 --- /dev/null +++ b/test/left_recursion/standard/left_recursion_test.go @@ -0,0 +1,26 @@ +package leftrecursion + +import ( + "testing" +) + +func TestLeftRecursion(t *testing.T) { + t.Parallel() + + data := "7+10/2*-4+5*3%6-8*6" + res, err := Parse("", []byte(data)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + data, err) + } + str, ok := res.(string) + if !ok { + t.FailNow() + } + want := "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))" + if str != want { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", data, str, want) + } +} diff --git a/test/max_expr_cnt/maxexpr.peg b/test/max_expr_cnt/maxexpr.peg index c3e9dc82..49170b42 100644 --- a/test/max_expr_cnt/maxexpr.peg +++ b/test/max_expr_cnt/maxexpr.peg @@ -8,4 +8,4 @@ long_rule2 = long_rule3 long_rule3 = long_rule4 long_rule4 = long_rule5 long_rule5 = long_rule6 -long_rule6 = " " \ No newline at end of file +long_rule6 = " " diff --git a/testutils/testutils.go b/testutils/testutils.go new file mode 100644 index 00000000..0b94b117 --- /dev/null +++ b/testutils/testutils.go @@ -0,0 +1,128 @@ +package testutils + +import ( + "bytes" + "reflect" +) + +// Copied from https://github.com/stretchr/testify + +// Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. All rights reserved. +// Use of this source code is governed by an MIT-style license that can be found in +// the THIRD-PARTY-NOTICES file. + +// isEmpty gets whether the specified object is considered empty or not. +func isEmpty(object interface{}) bool { + // get nil case out of the way + if object == nil { + return true + } + + objValue := reflect.ValueOf(object) + + switch objValue.Kind() { + // collection types are empty when they have no element + case reflect.Chan, reflect.Map, reflect.Slice: + return objValue.Len() == 0 + // pointers are empty if nil or if the value they point to is empty + case reflect.Ptr: + if objValue.IsNil() { + return true + } + deref := objValue.Elem().Interface() + return isEmpty(deref) + // for all other types, compare against the zero value + // array types are empty when they match their zero-initialized state + default: + zero := reflect.Zero(objValue.Type()) + return reflect.DeepEqual(object, zero.Interface()) + } +} + +// isList checks that the provided value is array or slice. +func isList(list interface{}) (ok bool) { + kind := reflect.TypeOf(list).Kind() + return kind == reflect.Array || kind == reflect.Slice +} + +// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. +// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and +// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. +func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { + aValue := reflect.ValueOf(listA) + bValue := reflect.ValueOf(listB) + + aLen := aValue.Len() + bLen := bValue.Len() + + // Mark indexes in bValue that we already used + visited := make([]bool, bLen) + for i := 0; i < aLen; i++ { + element := aValue.Index(i).Interface() + found := false + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + if ObjectsAreEqual(bValue.Index(j).Interface(), element) { + visited[j] = true + found = true + break + } + } + if !found { + extraA = append(extraA, element) + } + } + + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + extraB = append(extraB, bValue.Index(j).Interface()) + } + + return +} + +// ObjectsAreEqual determines if two objects are considered equal. +// +// This function does no assertion of any kind. +func ObjectsAreEqual(expected, actual interface{}) bool { + if expected == nil || actual == nil { + return expected == actual + } + + exp, ok := expected.([]byte) + if !ok { + return reflect.DeepEqual(expected, actual) + } + + act, ok := actual.([]byte) + if !ok { + return false + } + if exp == nil || act == nil { + return exp == nil && act == nil + } + return bytes.Equal(exp, act) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]). +func ElementsMatch(listA interface{}, listB interface{}) bool { + if isEmpty(listA) && isEmpty(listB) { + return true + } + + if !isList(listA) || !isList(listB) { + return false + } + + extraA, extraB := diffLists(listA, listB) + + return len(extraA) == 0 && len(extraB) == 0 +} From cd17ff23f2563ff03509d8227f5e6efe0a526a13 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Wed, 5 Jul 2023 18:04:30 +0300 Subject: [PATCH 08/14] Fix comments --- Makefile | 27 +- .../cmd/bootstrap-pigeon/bootstrap_pigeon.go | 20 +- builder/builder.go | 2 +- builder/generated_static_code.go | 120 +- builder/static_code.go | 120 +- doc.go | 64 +- examples/calculator/calculator.go | 20 +- examples/indentation/indentation.go | 20 +- examples/json/json.go | 20 +- examples/json/optimized-grammar/json.go | 20 +- examples/json/optimized/json.go | 18 +- pigeon.go | 20 +- targeted_test.go | 3 +- test/alternate_entrypoint/altentry.go | 20 +- test/andnot/andnot.go | 20 +- test/emptystate/emptystate.go | 20 +- test/errorpos/errorpos.go | 20 +- test/global_store/global_store.go | 20 +- test/goto/goto.go | 20 +- test/goto_state/goto_state.go | 20 +- test/issue_1/issue_1.go | 20 +- test/issue_18/issue_18.go | 20 +- test/issue_65/issue_65.go | 20 +- test/issue_65/optimized-grammar/issue_65.go | 20 +- test/issue_65/optimized/issue_65.go | 18 +- test/issue_70/issue_70.go | 20 +- test/issue_70/optimized-grammar/issue_70.go | 20 +- test/issue_70/optimized/issue_70.go | 18 +- test/issue_70b/issue_70b.go | 103 +- test/issue_80/issue_80.go | 20 +- test/labeled_failures/labeled_failures.go | 20 +- test/left_recursion/left_recursion.peg | 29 +- test/left_recursion/left_recursion_test.go | 197 +++ .../optimized/left_recursion_test.go | 26 - .../{ => leftrecursion}/left_recursion.go | 324 ++--- .../without_left_recursion.go | 1295 +++++++++++++++++ .../standard/left_recursion_test.go | 26 - .../leftrecursion}/left_recursion.go | 326 ++--- .../without_left_recursion.go} | 459 +++--- .../left_recursion/without_left_recursion.peg | 42 + test/linear/linear.go | 20 +- test/max_expr_cnt/maxexpr.go | 20 +- test/predicates/predicates.go | 20 +- test/runeerror/runeerror.go | 20 +- test/state/state.go | 20 +- test/stateclone/stateclone.go | 20 +- test/statereadonly/statereadonly.go | 20 +- test/staterestore/optimized/staterestore.go | 18 +- test/staterestore/standard/staterestore.go | 20 +- test/staterestore/staterestore.go | 20 +- test/thrownrecover/thrownrecover.go | 20 +- testutils/testutils.go | 68 +- testutils/testutils_test.go | 303 ++++ 53 files changed, 2606 insertions(+), 1620 deletions(-) create mode 100644 test/left_recursion/left_recursion_test.go delete mode 100644 test/left_recursion/optimized/left_recursion_test.go rename test/left_recursion/optimized/{ => leftrecursion}/left_recursion.go (85%) create mode 100644 test/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go delete mode 100644 test/left_recursion/standard/left_recursion_test.go rename test/left_recursion/{ => standart/leftrecursion}/left_recursion.go (87%) rename test/left_recursion/{standard/left_recursion.go => standart/withoutleftrecursion/without_left_recursion.go} (80%) create mode 100644 test/left_recursion/without_left_recursion.peg create mode 100644 testutils/testutils_test.go diff --git a/Makefile b/Makefile index 161ce897..c0a8c914 100644 --- a/Makefile +++ b/Makefile @@ -172,15 +172,34 @@ $(TEST_DIR)/issue_70b/issue_70b.go: $(TEST_DIR)/issue_70b/issue_70b.peg $(BINDIR $(TEST_DIR)/issue_80/issue_80.go: $(TEST_DIR)/issue_80/issue_80.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ -$(TEST_DIR)/left_recursion/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(TEST_DIR)/left_recursion/standard/left_recursion.go $(TEST_DIR)/left_recursion/optimized/left_recursion.go $(BINDIR)/pigeon - $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ +$(TEST_DIR)/left_recursion/left_recursion.go: \ + $(TEST_DIR)/left_recursion/standart/leftrecursion/left_recursion.go \ + $(TEST_DIR)/left_recursion/optimized/leftrecursion/left_recursion.go \ + $(BINDIR)/pigeon -$(TEST_DIR)/left_recursion/standard/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon +$(TEST_DIR)/left_recursion/standart/leftrecursion/left_recursion.go: \ + $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ -$(TEST_DIR)/left_recursion/optimized/left_recursion.go: $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon +$(TEST_DIR)/left_recursion/optimized/leftrecursion/left_recursion.go: \ + $(TEST_DIR)/left_recursion/left_recursion.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint -optimize-parser -support-left-recursion $< > $@ +$(TEST_DIR)/left_recursion/without_left_recursion.go: \ + $(TEST_DIR)/left_recursion/standart/withoutleftrecursion/without_left_recursion.go \ + $(TEST_DIR)/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go \ + $(BINDIR)/pigeon + +$(TEST_DIR)/left_recursion/standart/withoutleftrecursion/without_left_recursion.go: \ + $(TEST_DIR)/left_recursion/without_left_recursion.peg \ + $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint $< > $@ + +$(TEST_DIR)/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go: \ + $(TEST_DIR)/left_recursion/without_left_recursion.peg \ + $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -optimize-parser $< > $@ + lint: golint ./... diff --git a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go index 27ee02e2..55b660d1 100644 --- a/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go +++ b/bootstrap/cmd/bootstrap-pigeon/bootstrap_pigeon.go @@ -2765,18 +2765,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -2844,7 +2832,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -3103,11 +3091,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -3322,7 +3310,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/builder/builder.go b/builder/builder.go index 4d5b7d38..7767f55e 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -203,7 +203,7 @@ func (b *builder) writeRule(r *ast.Rule) { b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off) b.writef("\texpr: ") b.writeExpr(r.Expr) - if b.supportLeftRecursion { + if b.haveLeftRecursion { b.writelnf("\tleader: %t,", r.Leader) b.writelnf("\tleftRecursive: %t,", r.LeftRecursive) } diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index 6b067457..435018c0 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -529,11 +529,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - // ==template== {{ if .LeftRecursion }} - rstack []ruleWithExpsStack - // {{ else }} rstack []*rule - // {{ end }} ==template== // parse fail maxFailPos position @@ -586,47 +582,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - // ==template== {{ if .LeftRecursion }} - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) - // {{ else }} - p.rstack = append(p.rstack, rule) - // {{ end }} ==template== -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - // ==template== {{ if .LeftRecursion }} - return p.rstack[len(p.rstack)-1].rule - // {{ else }} - return p.rstack[len(p.rstack)-1] - // {{ end }} ==template== -} - -// ==template== {{ if .LeftRecursion }} -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - -// {{ end }} ==template== - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -697,7 +652,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -937,70 +892,9 @@ func listJoin(list []string, sep string, lastSep string) string { // ==template== {{ if .LeftRecursion }} -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } p.restore(result.end) return result.v, result.b } @@ -1125,11 +1019,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1164,9 +1058,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - // ==template== {{ if .LeftRecursion }} - p.pushExpr(expr) - // {{ end }} ==template== var val any var ok bool switch expr := expr.(type) { @@ -1211,9 +1102,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - // ==template== {{ if .LeftRecursion }} - p.popExpr() - // {{ end }} ==template== return val, ok } @@ -1396,7 +1284,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/builder/static_code.go b/builder/static_code.go index bbe8c588..c4df7c17 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -547,11 +547,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - // ==template== {{ if .LeftRecursion }} - rstack []ruleWithExpsStack - // {{ else }} rstack []*rule - // {{ end }} ==template== // parse fail maxFailPos position @@ -604,47 +600,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - // ==template== {{ if .LeftRecursion }} - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) - // {{ else }} - p.rstack = append(p.rstack, rule) - // {{ end }} ==template== -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - // ==template== {{ if .LeftRecursion }} - return p.rstack[len(p.rstack)-1].rule - // {{ else }} - return p.rstack[len(p.rstack)-1] - // {{ end }} ==template== -} - -// ==template== {{ if .LeftRecursion }} -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - -// {{ end }} ==template== - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -715,7 +670,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -955,70 +910,9 @@ func listJoin(list []string, sep string, lastSep string) string { // ==template== {{ if .LeftRecursion }} -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } p.restore(result.end) return result.v, result.b } @@ -1143,11 +1037,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1182,9 +1076,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - // ==template== {{ if .LeftRecursion }} - p.pushExpr(expr) - // {{ end }} ==template== var val any var ok bool switch expr := expr.(type) { @@ -1229,9 +1120,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - // ==template== {{ if .LeftRecursion }} - p.popExpr() - // {{ end }} ==template== return val, ok } @@ -1414,7 +1302,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { // ==template== {{ if not .Optimize }} func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/doc.go b/doc.go index 360c1ca9..e7842acb 100644 --- a/doc.go +++ b/doc.go @@ -177,7 +177,7 @@ on the following: For terminals (character and string literals, character classes and the any matcher), the value is []byte. E.g.: - Rule = label:'a' { // label is []byte } + Rule = label:'a' { // label is []byte } For predicates (& and !), the value is always nil. E.g.: Rule = label:&'a' { // label is nil } @@ -331,13 +331,13 @@ If a non-nil error is returned, it is added to the list of errors that the parser will return, note that the parser does NOT backtrack if a non-nil error is returned. E.g: - Rule = [a] #{ - c.state["a"]++ - if c.state["a"] > 5 { - return fmt.Errorf("we have seen more than 5 a's") // parser will not backtrack - } - return nil - } + Rule = [a] #{ + c.state["a"]++ + if c.state["a"] > 5 { + return fmt.Errorf("we have seen more than 5 a's") // parser will not backtrack + } + return nil + } The "*current" type is a struct that provides four useful fields that can be accessed in action, state change, and predicate code blocks: "pos", "text", "state" and "globalStore". @@ -395,15 +395,13 @@ With options -support-left-recursion pigeon supports left recursion. E.g.: Supports indirect recursion: A = B / D B = A / C -Preserve support for ordered choices: - expr = expr '*' term / expr '+' term The implementation is based on the [Left-recursive PEG Grammars][9] article that links to [Left Recursion in Parsing Expression Grammars][10] and [Packrat Parsers Can Support Left Recursion][11] papers. References: - [9]: https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1 + [9]: https://medium.com/@gvanrossum_83706/left-recursive-peg-grammars-65dab3c580e1 [10]: https://arxiv.org/pdf/1207.0443.pdf [11]: http://web.cs.ucla.edu/~todd/research/pepm08.pdf @@ -574,32 +572,32 @@ as the Go 1 compatibility [5]. The following lists what part of the current pigeon code falls under that guarantee (features may be added in the future): - - The pigeon command-line flags and arguments: those will not be removed - and will maintain the same semantics. + - The pigeon command-line flags and arguments: those will not be removed + and will maintain the same semantics. - - The explicitly exported API generated by pigeon. See [6] for the - documentation of this API on a generated parser. + - The explicitly exported API generated by pigeon. See [6] for the + documentation of this API on a generated parser. - - The PEG syntax, as documented above. + - The PEG syntax, as documented above. - - The code blocks (except the initializer) will always be generated as - methods on the *current type, and this type is guaranteed to have - the fields pos (type position) and text (type []byte). There are no - guarantees on other fields and methods of this type. + - The code blocks (except the initializer) will always be generated as + methods on the *current type, and this type is guaranteed to have + the fields pos (type position) and text (type []byte). There are no + guarantees on other fields and methods of this type. - - The position type will always have the fields line, col and offset, - all defined as int. There are no guarantees on other fields and methods - of this type. + - The position type will always have the fields line, col and offset, + all defined as int. There are no guarantees on other fields and methods + of this type. - - The type of the error value returned by the Parse* functions, when - not nil, will always be errList defined as a []error. There are no - guarantees on methods of this type, other than the fact it implements the - error interface. + - The type of the error value returned by the Parse* functions, when + not nil, will always be errList defined as a []error. There are no + guarantees on methods of this type, other than the fact it implements the + error interface. - - Individual errors in the errList will always be of type *parserError, - and this type is guaranteed to have an Inner field that contains the - original error value. There are no guarantees on other fields and methods - of this type. + - Individual errors in the errList will always be of type *parserError, + and this type is guaranteed to have an Inner field that contains the + original error value. There are no guarantees on other fields and methods + of this type. The above guarantee is given to the version 1.0 (https://github.com/mna/pigeon/releases/tag/v1.0.0) of pigeon, which has entered maintenance mode (bug fixes only). The current @@ -612,8 +610,8 @@ may occur at any time. References: - [5]: https://golang.org/doc/go1compat - [6]: http://godoc.org/github.com/mna/pigeon/test/predicates + [5]: https://golang.org/doc/go1compat + [6]: http://godoc.org/github.com/mna/pigeon/test/predicates */ package main diff --git a/examples/calculator/calculator.go b/examples/calculator/calculator.go index ed5147c7..a9e484d8 100644 --- a/examples/calculator/calculator.go +++ b/examples/calculator/calculator.go @@ -999,18 +999,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1078,7 +1066,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1338,11 +1326,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1559,7 +1547,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/indentation/indentation.go b/examples/indentation/indentation.go index 82ce0254..acab50bf 100644 --- a/examples/indentation/indentation.go +++ b/examples/indentation/indentation.go @@ -1312,18 +1312,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1391,7 +1379,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1651,11 +1639,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1872,7 +1860,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/json.go b/examples/json/json.go index c5b71757..c7c1b33e 100644 --- a/examples/json/json.go +++ b/examples/json/json.go @@ -1294,18 +1294,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1373,7 +1361,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1633,11 +1621,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1854,7 +1842,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized-grammar/json.go b/examples/json/optimized-grammar/json.go index f601a98a..bda15bde 100644 --- a/examples/json/optimized-grammar/json.go +++ b/examples/json/optimized-grammar/json.go @@ -1471,18 +1471,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1550,7 +1538,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1810,11 +1798,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -2031,7 +2019,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/examples/json/optimized/json.go b/examples/json/optimized/json.go index dfde3d88..5e6ae778 100644 --- a/examples/json/optimized/json.go +++ b/examples/json/optimized/json.go @@ -1216,18 +1216,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1270,7 +1258,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1426,11 +1414,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } diff --git a/pigeon.go b/pigeon.go index f54086be..a49c5c2f 100644 --- a/pigeon.go +++ b/pigeon.go @@ -3750,18 +3750,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -3829,7 +3817,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -4089,11 +4077,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -4310,7 +4298,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/targeted_test.go b/targeted_test.go index 932f8ff2..582d7d5e 100644 --- a/targeted_test.go +++ b/targeted_test.go @@ -765,8 +765,7 @@ func TestParseChoiceExpr(t *testing.T) { p := newParser("", []byte(tc.in)) // add dummy rule to rule stack of parser - r := rule{name: "dummy"} - p.pushRule(&r) + p.rstack = append(p.rstack, &rule{name: "dummy"}) // advance to the first rune p.read() diff --git a/test/alternate_entrypoint/altentry.go b/test/alternate_entrypoint/altentry.go index e2ad4f10..6945c161 100644 --- a/test/alternate_entrypoint/altentry.go +++ b/test/alternate_entrypoint/altentry.go @@ -768,18 +768,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -847,7 +835,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1107,11 +1095,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1328,7 +1316,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/andnot/andnot.go b/test/andnot/andnot.go index a6fa33ac..98fad741 100644 --- a/test/andnot/andnot.go +++ b/test/andnot/andnot.go @@ -708,18 +708,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -787,7 +775,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1047,11 +1035,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1268,7 +1256,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/emptystate/emptystate.go b/test/emptystate/emptystate.go index 9bd4512d..8f1ec330 100644 --- a/test/emptystate/emptystate.go +++ b/test/emptystate/emptystate.go @@ -768,18 +768,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -847,7 +835,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1107,11 +1095,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1328,7 +1316,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/errorpos/errorpos.go b/test/errorpos/errorpos.go index 87e47060..f52b5985 100644 --- a/test/errorpos/errorpos.go +++ b/test/errorpos/errorpos.go @@ -1147,18 +1147,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1226,7 +1214,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1486,11 +1474,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1707,7 +1695,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/global_store/global_store.go b/test/global_store/global_store.go index b08f8c3f..5cce23f2 100644 --- a/test/global_store/global_store.go +++ b/test/global_store/global_store.go @@ -729,18 +729,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -808,7 +796,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1068,11 +1056,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1289,7 +1277,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto/goto.go b/test/goto/goto.go index 63e3f3bc..f6b165e8 100644 --- a/test/goto/goto.go +++ b/test/goto/goto.go @@ -946,18 +946,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1025,7 +1013,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1285,11 +1273,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1506,7 +1494,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/goto_state/goto_state.go b/test/goto_state/goto_state.go index 3931234d..a4a10f2d 100644 --- a/test/goto_state/goto_state.go +++ b/test/goto_state/goto_state.go @@ -974,18 +974,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1053,7 +1041,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1313,11 +1301,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1534,7 +1522,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_1/issue_1.go b/test/issue_1/issue_1.go index 0eefbe6a..aa6803c8 100644 --- a/test/issue_1/issue_1.go +++ b/test/issue_1/issue_1.go @@ -648,18 +648,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -727,7 +715,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -987,11 +975,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1208,7 +1196,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_18/issue_18.go b/test/issue_18/issue_18.go index d55f1a94..087e7a20 100644 --- a/test/issue_18/issue_18.go +++ b/test/issue_18/issue_18.go @@ -665,18 +665,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -744,7 +732,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1004,11 +992,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1225,7 +1213,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/issue_65.go b/test/issue_65/issue_65.go index 05478369..de35eba6 100644 --- a/test/issue_65/issue_65.go +++ b/test/issue_65/issue_65.go @@ -664,18 +664,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -743,7 +731,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1003,11 +991,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1224,7 +1212,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized-grammar/issue_65.go b/test/issue_65/optimized-grammar/issue_65.go index 93aa0838..b828da0d 100644 --- a/test/issue_65/optimized-grammar/issue_65.go +++ b/test/issue_65/optimized-grammar/issue_65.go @@ -650,18 +650,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -729,7 +717,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -989,11 +977,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1210,7 +1198,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_65/optimized/issue_65.go b/test/issue_65/optimized/issue_65.go index 818bd81e..6d5717b3 100644 --- a/test/issue_65/optimized/issue_65.go +++ b/test/issue_65/optimized/issue_65.go @@ -579,18 +579,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -633,7 +621,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -789,11 +777,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } diff --git a/test/issue_70/issue_70.go b/test/issue_70/issue_70.go index 0c2791d6..cc80b506 100644 --- a/test/issue_70/issue_70.go +++ b/test/issue_70/issue_70.go @@ -639,18 +639,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -718,7 +706,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -978,11 +966,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1199,7 +1187,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized-grammar/issue_70.go b/test/issue_70/optimized-grammar/issue_70.go index b4179370..08212657 100644 --- a/test/issue_70/optimized-grammar/issue_70.go +++ b/test/issue_70/optimized-grammar/issue_70.go @@ -623,18 +623,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -702,7 +690,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -962,11 +950,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1183,7 +1171,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_70/optimized/issue_70.go b/test/issue_70/optimized/issue_70.go index ba3e22ae..09b95335 100644 --- a/test/issue_70/optimized/issue_70.go +++ b/test/issue_70/optimized/issue_70.go @@ -554,18 +554,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -608,7 +596,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -764,11 +752,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index 47208a62..730bd0ef 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -583,7 +583,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -636,36 +636,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1].rule -} - -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -733,7 +703,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -956,70 +926,9 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } p.restore(result.end) return result.v, result.b } @@ -1105,11 +1014,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1140,7 +1049,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1183,7 +1091,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1328,7 +1235,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/issue_80/issue_80.go b/test/issue_80/issue_80.go index 170de2b9..404f7e7d 100644 --- a/test/issue_80/issue_80.go +++ b/test/issue_80/issue_80.go @@ -658,18 +658,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -737,7 +725,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -997,11 +985,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1218,7 +1206,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/labeled_failures/labeled_failures.go b/test/labeled_failures/labeled_failures.go index 5daf7058..de8be5b3 100644 --- a/test/labeled_failures/labeled_failures.go +++ b/test/labeled_failures/labeled_failures.go @@ -887,18 +887,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -966,7 +954,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1226,11 +1214,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1447,7 +1435,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg index cba32c74..792638fa 100644 --- a/test/left_recursion/left_recursion.peg +++ b/test/left_recursion/left_recursion.peg @@ -5,21 +5,34 @@ start = a:expr !. { return a, nil } -expr = '-' a:expr { + +expr = a:expr op:('+'/'-') b:term { strA := a.(string) - return "(" + "-" + strA + ")", nil -} / a:expr op:('*' / '/' / '%') b:expr { + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil +} / a:term { + strA := a.(string) + return strA, nil +} + +term = a:term op:('*'/'/'/'%') b:factor { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil -} / a:expr op:('+' / '-') b:expr { + +} / a:factor { + strA := a.(string) + return strA, nil +} + +factor = op:('+' / '-') a:factor { strA := a.(string) - strB := b.(string) strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil -} / term { + return "(" + strOp + strA + ")", nil +} / atom { return string(c.text), nil } -term = [0-9]+ +atom = [0-9]+ \ No newline at end of file diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/left_recursion_test.go new file mode 100644 index 00000000..aa31c22b --- /dev/null +++ b/test/left_recursion/left_recursion_test.go @@ -0,0 +1,197 @@ +package leftrecursion_test + +import ( + "testing" + + "github.com/mna/pigeon/test/left_recursion/standart/leftrecursion" + "github.com/mna/pigeon/test/left_recursion/standart/withoutleftrecursion" + + optimizedleftrecursion "github.com/mna/pigeon/test/left_recursion/optimized/leftrecursion" + optimizedwithoutleftrecursion "github.com/mna/pigeon/test/left_recursion/optimized/withoutleftrecursion" +) + +func TestLeftRecursionParse(t *testing.T) { + t.Parallel() + + type want struct { + expr string + } + + tests := []struct { + name string + expr string + want want + }{ + { + name: "Complex", + expr: "7+10/2*-4+5*3%6-8*6", + want: want{expr: "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))"}, + }, + { + name: "Simple", + expr: "2*1+7", + want: want{expr: "((2*1)+7)"}, + }, + { + name: "Simple revers", + expr: "2+1*7", + want: want{expr: "(2+(1*7))"}, + }, + { + name: "Same operations", + expr: "2+1+7", + want: want{expr: "((2+1)+7)"}, + }, + { + name: "Start with unary minus", + expr: "-2+1", + want: want{expr: "((-2)+1)"}, + }, + { + name: "unary minus between + and *", + expr: "2+-7*-1", + want: want{expr: "(2+((-7)*(-1)))"}, + }, + } + + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name+" default", func(t *testing.T) { + t.Parallel() + + resLR, err := leftrecursion.Parse("", []byte(testCase.expr)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + exprLR, ok := resLR.(string) + if !ok { + t.FailNow() + } + if exprLR != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, exprLR, testCase.want.expr) + } + res, err := withoutleftrecursion.Parse("", []byte(testCase.expr)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, expr, testCase.want.expr) + } + }) + + t.Run(testCase.name+" optimezed", func(t *testing.T) { + t.Parallel() + + resLR, err := optimizedleftrecursion.Parse("", []byte(testCase.expr)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + exprLR, ok := resLR.(string) + if !ok { + t.FailNow() + } + if exprLR != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, exprLR, testCase.want.expr) + } + res, err := optimizedwithoutleftrecursion.Parse("", []byte(testCase.expr)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, expr, testCase.want.expr) + } + }) + } +} + +func FuzzLeftRecursionParse(f *testing.F) { + chars := []byte("0123456789+-/*%") + + f.Fuzz(func(t *testing.T, bytes []byte) { + data := make([]byte, 0, len(bytes)) + for _, b := range bytes { + data = append(data, chars[int(b)%len(chars)]) + } + resLR, errLR := leftrecursion.Parse("", data) + res, err := withoutleftrecursion.Parse("", data) + if err != nil || errLR != nil { + if err == nil || errLR == nil { + t.Fatalf( + "for input %q\ngot error: %q,\nbut expect: %q", + data, errLR, err) + } + return + } + exprLR, okLR := resLR.(string) + if !okLR { + t.FailNow() + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != exprLR { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + data, exprLR, expr) + } + }) +} + +func FuzzLeftRecursionParseOptimized(f *testing.F) { + chars := []byte("0123456789+-/*%") + + f.Fuzz(func(t *testing.T, bytes []byte) { + data := make([]byte, 0, len(bytes)) + for _, b := range bytes { + data = append(data, chars[int(b)%len(chars)]) + } + resLR, errLR := optimizedleftrecursion.Parse("", data) + res, err := optimizedwithoutleftrecursion.Parse("", data) + if err != nil || errLR != nil { + if err == nil || errLR == nil { + t.Fatalf( + "for input %q\ngot error: %q,\nbut expect: %q", + data, errLR, err) + } + return + } + exprLR, okLR := resLR.(string) + if !okLR { + t.FailNow() + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != exprLR { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + data, exprLR, expr) + } + }) +} diff --git a/test/left_recursion/optimized/left_recursion_test.go b/test/left_recursion/optimized/left_recursion_test.go deleted file mode 100644 index 63412630..00000000 --- a/test/left_recursion/optimized/left_recursion_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package leftrecursion - -import ( - "testing" -) - -func TestLeftRecursion(t *testing.T) { - t.Parallel() - - data := "7+10/2*-4+5*3%6-8*6" - res, err := Parse("", []byte(data)) - if err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - data, err) - } - str, ok := res.(string) - if !ok { - t.FailNow() - } - want := "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))" - if str != want { - t.Fatalf( - "for input %q\ngot result: %q,\nbut expect: %q", data, str, want) - } -} diff --git a/test/left_recursion/optimized/left_recursion.go b/test/left_recursion/optimized/leftrecursion/left_recursion.go similarity index 85% rename from test/left_recursion/optimized/left_recursion.go rename to test/left_recursion/optimized/leftrecursion/left_recursion.go index 599cf1b2..6a373f4e 100644 --- a/test/left_recursion/optimized/left_recursion.go +++ b/test/left_recursion/optimized/leftrecursion/left_recursion.go @@ -49,67 +49,113 @@ var g = &grammar{ }, { name: "expr", - pos: position{line: 8, col: 1, offset: 66}, + pos: position{line: 9, col: 1, offset: 67}, expr: &choiceExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, alternatives: []any{ &actionExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, run: (*parser).callonexpr2, expr: &seqExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, exprs: []any{ - &litMatcher{ - pos: position{line: 8, col: 8, offset: 73}, - val: "-", - ignoreCase: false, - want: "\"-\"", - }, &labeledExpr{ - pos: position{line: 8, col: 12, offset: 77}, + pos: position{line: 9, col: 9, offset: 75}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 8, col: 14, offset: 79}, + pos: position{line: 9, col: 11, offset: 77}, name: "expr", }, }, + &labeledExpr{ + pos: position{line: 9, col: 16, offset: 82}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 9, col: 20, offset: 86}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 9, col: 20, offset: 86}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 9, col: 24, offset: 90}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 9, col: 29, offset: 95}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 9, col: 31, offset: 97}, + name: "term", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 14, col: 5, offset: 235}, + run: (*parser).callonexpr12, + expr: &labeledExpr{ + pos: position{line: 14, col: 5, offset: 235}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 14, col: 7, offset: 237}, + name: "term", }, }, }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 19, col: 1, offset: 291}, + expr: &choiceExpr{ + pos: position{line: 19, col: 8, offset: 298}, + alternatives: []any{ &actionExpr{ - pos: position{line: 11, col: 5, offset: 152}, - run: (*parser).callonexpr7, + pos: position{line: 19, col: 8, offset: 298}, + run: (*parser).callonterm2, expr: &seqExpr{ - pos: position{line: 11, col: 5, offset: 152}, + pos: position{line: 19, col: 8, offset: 298}, exprs: []any{ &labeledExpr{ - pos: position{line: 11, col: 5, offset: 152}, + pos: position{line: 19, col: 8, offset: 298}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 11, col: 7, offset: 154}, - name: "expr", + pos: position{line: 19, col: 10, offset: 300}, + name: "term", }, }, &labeledExpr{ - pos: position{line: 11, col: 12, offset: 159}, + pos: position{line: 19, col: 15, offset: 305}, label: "op", expr: &choiceExpr{ - pos: position{line: 11, col: 16, offset: 163}, + pos: position{line: 19, col: 19, offset: 309}, alternatives: []any{ &litMatcher{ - pos: position{line: 11, col: 16, offset: 163}, + pos: position{line: 19, col: 19, offset: 309}, val: "*", ignoreCase: false, want: "\"*\"", }, &litMatcher{ - pos: position{line: 11, col: 22, offset: 169}, + pos: position{line: 19, col: 23, offset: 313}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 11, col: 28, offset: 175}, + pos: position{line: 19, col: 27, offset: 317}, val: "%", ignoreCase: false, want: "\"%\"", @@ -118,44 +164,59 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 11, col: 33, offset: 180}, + pos: position{line: 19, col: 32, offset: 322}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 11, col: 35, offset: 182}, - name: "expr", + pos: position{line: 19, col: 34, offset: 324}, + name: "factor", }, }, }, }, }, &actionExpr{ - pos: position{line: 16, col: 5, offset: 321}, - run: (*parser).callonexpr18, + pos: position{line: 25, col: 5, offset: 466}, + run: (*parser).callonterm13, + expr: &labeledExpr{ + pos: position{line: 25, col: 5, offset: 466}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 25, col: 7, offset: 468}, + name: "factor", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "factor", + pos: position{line: 30, col: 1, offset: 524}, + expr: &choiceExpr{ + pos: position{line: 30, col: 10, offset: 533}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 30, col: 10, offset: 533}, + run: (*parser).callonfactor2, expr: &seqExpr{ - pos: position{line: 16, col: 5, offset: 321}, + pos: position{line: 30, col: 10, offset: 533}, exprs: []any{ &labeledExpr{ - pos: position{line: 16, col: 5, offset: 321}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 16, col: 7, offset: 323}, - name: "expr", - }, - }, - &labeledExpr{ - pos: position{line: 16, col: 12, offset: 328}, + pos: position{line: 30, col: 10, offset: 533}, label: "op", expr: &choiceExpr{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 30, col: 14, offset: 537}, alternatives: []any{ &litMatcher{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 30, col: 14, offset: 537}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 16, col: 22, offset: 338}, + pos: position{line: 30, col: 20, offset: 543}, val: "-", ignoreCase: false, want: "\"-\"", @@ -164,36 +225,36 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 16, col: 27, offset: 343}, - label: "b", + pos: position{line: 30, col: 25, offset: 548}, + label: "a", expr: &ruleRefExpr{ - pos: position{line: 16, col: 29, offset: 345}, - name: "expr", + pos: position{line: 30, col: 27, offset: 550}, + name: "factor", }, }, }, }, }, &actionExpr{ - pos: position{line: 21, col: 5, offset: 483}, - run: (*parser).callonexpr28, + pos: position{line: 34, col: 5, offset: 660}, + run: (*parser).callonfactor10, expr: &ruleRefExpr{ - pos: position{line: 21, col: 5, offset: 483}, - name: "term", + pos: position{line: 34, col: 5, offset: 660}, + name: "atom", }, }, }, }, - leader: true, - leftRecursive: true, + leader: false, + leftRecursive: false, }, { - name: "term", - pos: position{line: 25, col: 1, offset: 524}, + name: "atom", + pos: position{line: 38, col: 1, offset: 701}, expr: &oneOrMoreExpr{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 38, col: 8, offset: 708}, expr: &charClassMatcher{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 38, col: 8, offset: 708}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -216,51 +277,75 @@ func (p *parser) callonstart1() (any, error) { return p.cur.onstart1(stack["a"]) } -func (c *current) onexpr2(a any) (any, error) { +func (c *current) onexpr2(a, op, b any) (any, error) { strA := a.(string) - return "(" + "-" + strA + ")", nil + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil } func (p *parser) callonexpr2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr2(stack["a"]) + return p.cur.onexpr2(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr12(a any) (any, error) { + strA := a.(string) + return strA, nil +} + +func (p *parser) callonexpr12() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr12(stack["a"]) } -func (c *current) onexpr7(a, op, b any) (any, error) { +func (c *current) onterm2(a, op, b any) (any, error) { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil + } -func (p *parser) callonexpr7() (any, error) { +func (p *parser) callonterm2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) + return p.cur.onterm2(stack["a"], stack["op"], stack["b"]) } -func (c *current) onexpr18(a, op, b any) (any, error) { +func (c *current) onterm13(a any) (any, error) { + strA := a.(string) + return strA, nil +} + +func (p *parser) callonterm13() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm13(stack["a"]) +} + +func (c *current) onfactor2(op, a any) (any, error) { strA := a.(string) - strB := b.(string) strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil + return "(" + strOp + strA + ")", nil } -func (p *parser) callonexpr18() (any, error) { +func (p *parser) callonfactor2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) + return p.cur.onfactor2(stack["op"], stack["a"]) } -func (c *current) onexpr28() (any, error) { +func (c *current) onfactor10() (any, error) { return string(c.text), nil } -func (p *parser) callonexpr28() (any, error) { +func (p *parser) callonfactor10() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr28() + return p.cur.onfactor10() } var ( @@ -682,7 +767,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -735,36 +820,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1].rule -} - -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -807,7 +862,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -975,70 +1030,9 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } p.restore(result.end) return result.v, result.b } @@ -1090,11 +1084,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1111,7 +1105,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1152,7 +1145,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } diff --git a/test/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go b/test/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go new file mode 100644 index 00000000..749d6f88 --- /dev/null +++ b/test/left_recursion/optimized/withoutleftrecursion/without_left_recursion.go @@ -0,0 +1,1295 @@ +// Code generated by pigeon; DO NOT EDIT. + +package withoutleftrecursion + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +func toAnySlice(v any) []any { + if v == nil { + return nil + } + return v.([]any) +} + +func exprToString(first string, rest any) string { + restSl := toAnySlice(rest) + l := first + for _, v := range restSl { + restExpr := toAnySlice(v) + r := restExpr[1].(string) + op := string(restExpr[0].([]byte)) + l = "(" + l + op + r + ")" + } + return l +} + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 24, col: 1, offset: 500}, + expr: &actionExpr{ + pos: position{line: 24, col: 9, offset: 508}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 24, col: 9, offset: 508}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 24, col: 9, offset: 508}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 24, col: 11, offset: 510}, + name: "expr", + }, + }, + ¬Expr{ + pos: position{line: 24, col: 16, offset: 515}, + expr: &anyMatcher{ + line: 24, col: 17, offset: 516, + }, + }, + }, + }, + }, + }, + { + name: "expr", + pos: position{line: 27, col: 1, offset: 537}, + expr: &actionExpr{ + pos: position{line: 27, col: 8, offset: 544}, + run: (*parser).callonexpr1, + expr: &seqExpr{ + pos: position{line: 27, col: 8, offset: 544}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 27, col: 8, offset: 544}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 27, col: 10, offset: 546}, + name: "term", + }, + }, + &labeledExpr{ + pos: position{line: 27, col: 15, offset: 551}, + label: "b", + expr: &zeroOrMoreExpr{ + pos: position{line: 27, col: 17, offset: 553}, + expr: &seqExpr{ + pos: position{line: 27, col: 18, offset: 554}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 27, col: 20, offset: 556}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 27, col: 20, offset: 556}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 27, col: 26, offset: 562}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 27, col: 32, offset: 568}, + name: "term", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "term", + pos: position{line: 31, col: 1, offset: 641}, + expr: &actionExpr{ + pos: position{line: 31, col: 8, offset: 648}, + run: (*parser).callonterm1, + expr: &seqExpr{ + pos: position{line: 31, col: 8, offset: 648}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 31, col: 8, offset: 648}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 31, col: 10, offset: 650}, + name: "factor", + }, + }, + &labeledExpr{ + pos: position{line: 31, col: 17, offset: 657}, + label: "b", + expr: &zeroOrMoreExpr{ + pos: position{line: 31, col: 19, offset: 659}, + expr: &seqExpr{ + pos: position{line: 31, col: 21, offset: 661}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 31, col: 23, offset: 663}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 31, col: 23, offset: 663}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 31, col: 29, offset: 669}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 31, col: 35, offset: 675}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 31, col: 40, offset: 680}, + name: "factor", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "factor", + pos: position{line: 35, col: 1, offset: 755}, + expr: &choiceExpr{ + pos: position{line: 35, col: 10, offset: 764}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 35, col: 10, offset: 764}, + run: (*parser).callonfactor2, + expr: &seqExpr{ + pos: position{line: 35, col: 10, offset: 764}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 35, col: 10, offset: 764}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 35, col: 14, offset: 768}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 35, col: 14, offset: 768}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 35, col: 20, offset: 774}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 35, col: 25, offset: 779}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 35, col: 27, offset: 781}, + name: "factor", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 39, col: 5, offset: 891}, + run: (*parser).callonfactor10, + expr: &ruleRefExpr{ + pos: position{line: 39, col: 5, offset: 891}, + name: "atom", + }, + }, + }, + }, + }, + { + name: "atom", + pos: position{line: 42, col: 1, offset: 931}, + expr: &oneOrMoreExpr{ + pos: position{line: 42, col: 8, offset: 938}, + expr: &charClassMatcher{ + pos: position{line: 42, col: 8, offset: 938}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, +} + +func (c *current) onstart1(a any) (any, error) { + return a, nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1(stack["a"]) +} + +func (c *current) onexpr1(a, b any) (any, error) { + strA := a.(string) + return exprToString(strA, b), nil +} + +func (p *parser) callonexpr1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr1(stack["a"], stack["b"]) +} + +func (c *current) onterm1(a, b any) (any, error) { + strA := a.(string) + return exprToString(strA, b), nil +} + +func (p *parser) callonterm1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm1(stack["a"], stack["b"]) +} + +func (c *current) onfactor2(op, a any) (any, error) { + strA := a.(string) + strOp := string(op.([]byte)) + return "(" + strOp + strA + ")", nil +} + +func (p *parser) callonfactor2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor2(stack["op"], stack["a"]) +} + +func (c *current) onfactor10() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonfactor10() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor10() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + val, ok = p.parseRule(rule) + + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + + val = actVal + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + pt := p.pt + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + return val, ok + } + } + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + pt := p.pt + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion/standard/left_recursion_test.go b/test/left_recursion/standard/left_recursion_test.go deleted file mode 100644 index 63412630..00000000 --- a/test/left_recursion/standard/left_recursion_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package leftrecursion - -import ( - "testing" -) - -func TestLeftRecursion(t *testing.T) { - t.Parallel() - - data := "7+10/2*-4+5*3%6-8*6" - res, err := Parse("", []byte(data)) - if err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - data, err) - } - str, ok := res.(string) - if !ok { - t.FailNow() - } - want := "(((7+((10/2)*(-4)))+((5*3)%6))-(8*6))" - if str != want { - t.Fatalf( - "for input %q\ngot result: %q,\nbut expect: %q", data, str, want) - } -} diff --git a/test/left_recursion/left_recursion.go b/test/left_recursion/standart/leftrecursion/left_recursion.go similarity index 87% rename from test/left_recursion/left_recursion.go rename to test/left_recursion/standart/leftrecursion/left_recursion.go index a3dffeae..4c8a256b 100644 --- a/test/left_recursion/left_recursion.go +++ b/test/left_recursion/standart/leftrecursion/left_recursion.go @@ -50,67 +50,113 @@ var g = &grammar{ }, { name: "expr", - pos: position{line: 8, col: 1, offset: 66}, + pos: position{line: 9, col: 1, offset: 67}, expr: &choiceExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, alternatives: []any{ &actionExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, run: (*parser).callonexpr2, expr: &seqExpr{ - pos: position{line: 8, col: 8, offset: 73}, + pos: position{line: 9, col: 9, offset: 75}, exprs: []any{ - &litMatcher{ - pos: position{line: 8, col: 8, offset: 73}, - val: "-", - ignoreCase: false, - want: "\"-\"", - }, &labeledExpr{ - pos: position{line: 8, col: 12, offset: 77}, + pos: position{line: 9, col: 9, offset: 75}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 8, col: 14, offset: 79}, + pos: position{line: 9, col: 11, offset: 77}, name: "expr", }, }, + &labeledExpr{ + pos: position{line: 9, col: 16, offset: 82}, + label: "op", + expr: &choiceExpr{ + pos: position{line: 9, col: 20, offset: 86}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 9, col: 20, offset: 86}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 9, col: 24, offset: 90}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 9, col: 29, offset: 95}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 9, col: 31, offset: 97}, + name: "term", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 14, col: 5, offset: 235}, + run: (*parser).callonexpr12, + expr: &labeledExpr{ + pos: position{line: 14, col: 5, offset: 235}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 14, col: 7, offset: 237}, + name: "term", }, }, }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 19, col: 1, offset: 291}, + expr: &choiceExpr{ + pos: position{line: 19, col: 8, offset: 298}, + alternatives: []any{ &actionExpr{ - pos: position{line: 11, col: 5, offset: 152}, - run: (*parser).callonexpr7, + pos: position{line: 19, col: 8, offset: 298}, + run: (*parser).callonterm2, expr: &seqExpr{ - pos: position{line: 11, col: 5, offset: 152}, + pos: position{line: 19, col: 8, offset: 298}, exprs: []any{ &labeledExpr{ - pos: position{line: 11, col: 5, offset: 152}, + pos: position{line: 19, col: 8, offset: 298}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 11, col: 7, offset: 154}, - name: "expr", + pos: position{line: 19, col: 10, offset: 300}, + name: "term", }, }, &labeledExpr{ - pos: position{line: 11, col: 12, offset: 159}, + pos: position{line: 19, col: 15, offset: 305}, label: "op", expr: &choiceExpr{ - pos: position{line: 11, col: 16, offset: 163}, + pos: position{line: 19, col: 19, offset: 309}, alternatives: []any{ &litMatcher{ - pos: position{line: 11, col: 16, offset: 163}, + pos: position{line: 19, col: 19, offset: 309}, val: "*", ignoreCase: false, want: "\"*\"", }, &litMatcher{ - pos: position{line: 11, col: 22, offset: 169}, + pos: position{line: 19, col: 23, offset: 313}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 11, col: 28, offset: 175}, + pos: position{line: 19, col: 27, offset: 317}, val: "%", ignoreCase: false, want: "\"%\"", @@ -119,44 +165,59 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 11, col: 33, offset: 180}, + pos: position{line: 19, col: 32, offset: 322}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 11, col: 35, offset: 182}, - name: "expr", + pos: position{line: 19, col: 34, offset: 324}, + name: "factor", }, }, }, }, }, &actionExpr{ - pos: position{line: 16, col: 5, offset: 321}, - run: (*parser).callonexpr18, + pos: position{line: 25, col: 5, offset: 466}, + run: (*parser).callonterm13, + expr: &labeledExpr{ + pos: position{line: 25, col: 5, offset: 466}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 25, col: 7, offset: 468}, + name: "factor", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "factor", + pos: position{line: 30, col: 1, offset: 524}, + expr: &choiceExpr{ + pos: position{line: 30, col: 10, offset: 533}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 30, col: 10, offset: 533}, + run: (*parser).callonfactor2, expr: &seqExpr{ - pos: position{line: 16, col: 5, offset: 321}, + pos: position{line: 30, col: 10, offset: 533}, exprs: []any{ &labeledExpr{ - pos: position{line: 16, col: 5, offset: 321}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 16, col: 7, offset: 323}, - name: "expr", - }, - }, - &labeledExpr{ - pos: position{line: 16, col: 12, offset: 328}, + pos: position{line: 30, col: 10, offset: 533}, label: "op", expr: &choiceExpr{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 30, col: 14, offset: 537}, alternatives: []any{ &litMatcher{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 30, col: 14, offset: 537}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 16, col: 22, offset: 338}, + pos: position{line: 30, col: 20, offset: 543}, val: "-", ignoreCase: false, want: "\"-\"", @@ -165,36 +226,36 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 16, col: 27, offset: 343}, - label: "b", + pos: position{line: 30, col: 25, offset: 548}, + label: "a", expr: &ruleRefExpr{ - pos: position{line: 16, col: 29, offset: 345}, - name: "expr", + pos: position{line: 30, col: 27, offset: 550}, + name: "factor", }, }, }, }, }, &actionExpr{ - pos: position{line: 21, col: 5, offset: 483}, - run: (*parser).callonexpr28, + pos: position{line: 34, col: 5, offset: 660}, + run: (*parser).callonfactor10, expr: &ruleRefExpr{ - pos: position{line: 21, col: 5, offset: 483}, - name: "term", + pos: position{line: 34, col: 5, offset: 660}, + name: "atom", }, }, }, }, - leader: true, - leftRecursive: true, + leader: false, + leftRecursive: false, }, { - name: "term", - pos: position{line: 25, col: 1, offset: 524}, + name: "atom", + pos: position{line: 38, col: 1, offset: 701}, expr: &oneOrMoreExpr{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 38, col: 8, offset: 708}, expr: &charClassMatcher{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 38, col: 8, offset: 708}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -217,51 +278,75 @@ func (p *parser) callonstart1() (any, error) { return p.cur.onstart1(stack["a"]) } -func (c *current) onexpr2(a any) (any, error) { +func (c *current) onexpr2(a, op, b any) (any, error) { strA := a.(string) - return "(" + "-" + strA + ")", nil + strB := b.(string) + strOp := string(op.([]byte)) + return "(" + strA + strOp + strB + ")", nil } func (p *parser) callonexpr2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr2(stack["a"]) + return p.cur.onexpr2(stack["a"], stack["op"], stack["b"]) +} + +func (c *current) onexpr12(a any) (any, error) { + strA := a.(string) + return strA, nil +} + +func (p *parser) callonexpr12() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr12(stack["a"]) } -func (c *current) onexpr7(a, op, b any) (any, error) { +func (c *current) onterm2(a, op, b any) (any, error) { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) return "(" + strA + strOp + strB + ")", nil + } -func (p *parser) callonexpr7() (any, error) { +func (p *parser) callonterm2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) + return p.cur.onterm2(stack["a"], stack["op"], stack["b"]) } -func (c *current) onexpr18(a, op, b any) (any, error) { +func (c *current) onterm13(a any) (any, error) { + strA := a.(string) + return strA, nil +} + +func (p *parser) callonterm13() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm13(stack["a"]) +} + +func (c *current) onfactor2(op, a any) (any, error) { strA := a.(string) - strB := b.(string) strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil + return "(" + strOp + strA + ")", nil } -func (p *parser) callonexpr18() (any, error) { +func (p *parser) callonfactor2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) + return p.cur.onfactor2(stack["op"], stack["a"]) } -func (c *current) onexpr28() (any, error) { +func (c *current) onfactor10() (any, error) { return string(c.text), nil } -func (p *parser) callonexpr28() (any, error) { +func (p *parser) callonfactor10() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr28() + return p.cur.onfactor10() } var ( @@ -764,7 +849,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -817,36 +902,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1].rule -} - -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -914,7 +969,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1137,70 +1192,9 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { result, ok := p.getMemoized(rule) if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } p.restore(result.end) return result.v, result.b } @@ -1286,11 +1280,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1321,7 +1315,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1364,7 +1357,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1509,7 +1501,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/standard/left_recursion.go b/test/left_recursion/standart/withoutleftrecursion/without_left_recursion.go similarity index 80% rename from test/left_recursion/standard/left_recursion.go rename to test/left_recursion/standart/withoutleftrecursion/without_left_recursion.go index a3dffeae..51bfe024 100644 --- a/test/left_recursion/standard/left_recursion.go +++ b/test/left_recursion/standart/withoutleftrecursion/without_left_recursion.go @@ -1,6 +1,6 @@ // Code generated by pigeon; DO NOT EDIT. -package leftrecursion +package withoutleftrecursion import ( "bytes" @@ -17,146 +17,194 @@ import ( "unicode/utf8" ) +func toAnySlice(v any) []any { + if v == nil { + return nil + } + return v.([]any) +} + +func exprToString(first string, rest any) string { + restSl := toAnySlice(rest) + l := first + for _, v := range restSl { + restExpr := toAnySlice(v) + r := restExpr[1].(string) + op := string(restExpr[0].([]byte)) + l = "(" + l + op + r + ")" + } + return l +} + var g = &grammar{ rules: []*rule{ { name: "start", - pos: position{line: 5, col: 1, offset: 29}, + pos: position{line: 24, col: 1, offset: 500}, expr: &actionExpr{ - pos: position{line: 5, col: 9, offset: 37}, + pos: position{line: 24, col: 9, offset: 508}, run: (*parser).callonstart1, expr: &seqExpr{ - pos: position{line: 5, col: 9, offset: 37}, + pos: position{line: 24, col: 9, offset: 508}, exprs: []any{ &labeledExpr{ - pos: position{line: 5, col: 9, offset: 37}, + pos: position{line: 24, col: 9, offset: 508}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 5, col: 11, offset: 39}, + pos: position{line: 24, col: 11, offset: 510}, name: "expr", }, }, ¬Expr{ - pos: position{line: 5, col: 16, offset: 44}, + pos: position{line: 24, col: 16, offset: 515}, expr: &anyMatcher{ - line: 5, col: 17, offset: 45, + line: 24, col: 17, offset: 516, }, }, }, }, }, - leader: false, - leftRecursive: false, }, { name: "expr", - pos: position{line: 8, col: 1, offset: 66}, - expr: &choiceExpr{ - pos: position{line: 8, col: 8, offset: 73}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 8, col: 8, offset: 73}, - run: (*parser).callonexpr2, - expr: &seqExpr{ - pos: position{line: 8, col: 8, offset: 73}, - exprs: []any{ - &litMatcher{ - pos: position{line: 8, col: 8, offset: 73}, - val: "-", - ignoreCase: false, - want: "\"-\"", - }, - &labeledExpr{ - pos: position{line: 8, col: 12, offset: 77}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 8, col: 14, offset: 79}, - name: "expr", + pos: position{line: 27, col: 1, offset: 537}, + expr: &actionExpr{ + pos: position{line: 27, col: 8, offset: 544}, + run: (*parser).callonexpr1, + expr: &seqExpr{ + pos: position{line: 27, col: 8, offset: 544}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 27, col: 8, offset: 544}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 27, col: 10, offset: 546}, + name: "term", + }, + }, + &labeledExpr{ + pos: position{line: 27, col: 15, offset: 551}, + label: "b", + expr: &zeroOrMoreExpr{ + pos: position{line: 27, col: 17, offset: 553}, + expr: &seqExpr{ + pos: position{line: 27, col: 18, offset: 554}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 27, col: 20, offset: 556}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 27, col: 20, offset: 556}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 27, col: 26, offset: 562}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 27, col: 32, offset: 568}, + name: "term", + }, }, }, }, }, }, - &actionExpr{ - pos: position{line: 11, col: 5, offset: 152}, - run: (*parser).callonexpr7, - expr: &seqExpr{ - pos: position{line: 11, col: 5, offset: 152}, - exprs: []any{ - &labeledExpr{ - pos: position{line: 11, col: 5, offset: 152}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 11, col: 7, offset: 154}, - name: "expr", - }, - }, - &labeledExpr{ - pos: position{line: 11, col: 12, offset: 159}, - label: "op", - expr: &choiceExpr{ - pos: position{line: 11, col: 16, offset: 163}, - alternatives: []any{ - &litMatcher{ - pos: position{line: 11, col: 16, offset: 163}, - val: "*", - ignoreCase: false, - want: "\"*\"", - }, - &litMatcher{ - pos: position{line: 11, col: 22, offset: 169}, - val: "/", - ignoreCase: false, - want: "\"/\"", - }, - &litMatcher{ - pos: position{line: 11, col: 28, offset: 175}, - val: "%", - ignoreCase: false, - want: "\"%\"", + }, + }, + }, + { + name: "term", + pos: position{line: 31, col: 1, offset: 641}, + expr: &actionExpr{ + pos: position{line: 31, col: 8, offset: 648}, + run: (*parser).callonterm1, + expr: &seqExpr{ + pos: position{line: 31, col: 8, offset: 648}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 31, col: 8, offset: 648}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 31, col: 10, offset: 650}, + name: "factor", + }, + }, + &labeledExpr{ + pos: position{line: 31, col: 17, offset: 657}, + label: "b", + expr: &zeroOrMoreExpr{ + pos: position{line: 31, col: 19, offset: 659}, + expr: &seqExpr{ + pos: position{line: 31, col: 21, offset: 661}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 31, col: 23, offset: 663}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 31, col: 23, offset: 663}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 31, col: 29, offset: 669}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 31, col: 35, offset: 675}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, }, }, - }, - }, - &labeledExpr{ - pos: position{line: 11, col: 33, offset: 180}, - label: "b", - expr: &ruleRefExpr{ - pos: position{line: 11, col: 35, offset: 182}, - name: "expr", + &ruleRefExpr{ + pos: position{line: 31, col: 40, offset: 680}, + name: "factor", + }, }, }, }, }, }, + }, + }, + }, + { + name: "factor", + pos: position{line: 35, col: 1, offset: 755}, + expr: &choiceExpr{ + pos: position{line: 35, col: 10, offset: 764}, + alternatives: []any{ &actionExpr{ - pos: position{line: 16, col: 5, offset: 321}, - run: (*parser).callonexpr18, + pos: position{line: 35, col: 10, offset: 764}, + run: (*parser).callonfactor2, expr: &seqExpr{ - pos: position{line: 16, col: 5, offset: 321}, + pos: position{line: 35, col: 10, offset: 764}, exprs: []any{ &labeledExpr{ - pos: position{line: 16, col: 5, offset: 321}, - label: "a", - expr: &ruleRefExpr{ - pos: position{line: 16, col: 7, offset: 323}, - name: "expr", - }, - }, - &labeledExpr{ - pos: position{line: 16, col: 12, offset: 328}, + pos: position{line: 35, col: 10, offset: 764}, label: "op", expr: &choiceExpr{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 35, col: 14, offset: 768}, alternatives: []any{ &litMatcher{ - pos: position{line: 16, col: 16, offset: 332}, + pos: position{line: 35, col: 14, offset: 768}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 16, col: 22, offset: 338}, + pos: position{line: 35, col: 20, offset: 774}, val: "-", ignoreCase: false, want: "\"-\"", @@ -165,44 +213,40 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 16, col: 27, offset: 343}, - label: "b", + pos: position{line: 35, col: 25, offset: 779}, + label: "a", expr: &ruleRefExpr{ - pos: position{line: 16, col: 29, offset: 345}, - name: "expr", + pos: position{line: 35, col: 27, offset: 781}, + name: "factor", }, }, }, }, }, &actionExpr{ - pos: position{line: 21, col: 5, offset: 483}, - run: (*parser).callonexpr28, + pos: position{line: 39, col: 5, offset: 891}, + run: (*parser).callonfactor10, expr: &ruleRefExpr{ - pos: position{line: 21, col: 5, offset: 483}, - name: "term", + pos: position{line: 39, col: 5, offset: 891}, + name: "atom", }, }, }, }, - leader: true, - leftRecursive: true, }, { - name: "term", - pos: position{line: 25, col: 1, offset: 524}, + name: "atom", + pos: position{line: 42, col: 1, offset: 931}, expr: &oneOrMoreExpr{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 42, col: 8, offset: 938}, expr: &charClassMatcher{ - pos: position{line: 25, col: 8, offset: 531}, + pos: position{line: 42, col: 8, offset: 938}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, inverted: false, }, }, - leader: false, - leftRecursive: false, }, }, } @@ -217,51 +261,48 @@ func (p *parser) callonstart1() (any, error) { return p.cur.onstart1(stack["a"]) } -func (c *current) onexpr2(a any) (any, error) { +func (c *current) onexpr1(a, b any) (any, error) { strA := a.(string) - return "(" + "-" + strA + ")", nil + return exprToString(strA, b), nil } -func (p *parser) callonexpr2() (any, error) { +func (p *parser) callonexpr1() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr2(stack["a"]) + return p.cur.onexpr1(stack["a"], stack["b"]) } -func (c *current) onexpr7(a, op, b any) (any, error) { +func (c *current) onterm1(a, b any) (any, error) { strA := a.(string) - strB := b.(string) - strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil + return exprToString(strA, b), nil } -func (p *parser) callonexpr7() (any, error) { +func (p *parser) callonterm1() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr7(stack["a"], stack["op"], stack["b"]) + return p.cur.onterm1(stack["a"], stack["b"]) } -func (c *current) onexpr18(a, op, b any) (any, error) { +func (c *current) onfactor2(op, a any) (any, error) { strA := a.(string) - strB := b.(string) strOp := string(op.([]byte)) - return "(" + strA + strOp + strB + ")", nil + return "(" + strOp + strA + ")", nil } -func (p *parser) callonexpr18() (any, error) { +func (p *parser) callonfactor2() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr18(stack["a"], stack["op"], stack["b"]) + return p.cur.onfactor2(stack["op"], stack["a"]) } -func (c *current) onexpr28() (any, error) { +func (c *current) onfactor10() (any, error) { return string(c.text), nil } -func (p *parser) callonexpr28() (any, error) { +func (p *parser) callonfactor10() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onexpr28() + return p.cur.onfactor10() } var ( @@ -501,9 +542,6 @@ type rule struct { name string displayName string expr any - - leader bool - leftRecursive bool } // nolint: structcheck @@ -736,11 +774,6 @@ type Stats struct { ChoiceAltCnt map[string]map[string]int } -type ruleWithExpsStack struct { - rule *rule - estack []any -} - // nolint: structcheck,maligned type parser struct { filename string @@ -764,7 +797,7 @@ type parser struct { // variables stack, map of label to value vstack []map[string]any // rule stack, allows identification of the current rule in errors - rstack []ruleWithExpsStack + rstack []*rule // parse fail maxFailPos position @@ -817,36 +850,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, ruleWithExpsStack{rule: rule}) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1].rule -} - -func (p *parser) pushExpr(expr any) { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = append( - p.rstack[len(p.rstack)-1].estack, expr, - ) -} - -func (p *parser) popExpr() { - if len(p.rstack) == 0 { - return - } - p.rstack[len(p.rstack)-1].estack = p.rstack[len(p.rstack)-1].estack[:len( - p.rstack[len(p.rstack)-1].estack, - )-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -914,7 +917,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1137,112 +1140,6 @@ func listJoin(list []string, sep string, lastSep string) string { } } -func (p *parser) checkPrevChoice(rule *rule) (checkPriority, haveChoice bool) { - var currentEStack []any - var prevEStack []any - for i := 1; i <= len(p.rstack)/2; i++ { - indexCurrent := len(p.rstack) - i - indexPrev := len(p.rstack) - i*2 - if p.rstack[indexCurrent].rule == rule && p.rstack[indexPrev].rule == rule { - currentEStack = p.rstack[indexCurrent].estack - prevEStack = p.rstack[indexPrev].estack - break - } - } - if prevEStack == nil || currentEStack == nil { - return false, false - } - if len(prevEStack) != len(currentEStack) { - panic("Stacks are not equal(len)") - } - - for i := len(prevEStack) - 1; i >= 0; i-- { - currentCh, ok := currentEStack[i].(*choiceExpr) - if !ok { - continue - } - prevCh, ok := prevEStack[i].(*choiceExpr) - if !ok { - panic("Stacks are not equal(position choiceExpr)") - } - if currentCh != prevCh { - panic("Stacks are not equal(choiceExpr)") - } - currentAlt := -1 - for j, inExp := range currentCh.alternatives { - if inExp == currentEStack[i+1] { - currentAlt = j - break - } - } - if currentAlt == -1 { - panic("lost alternatives in choiceExpr") - } - prevAlt := -1 - for j, inExp := range prevCh.alternatives { - if inExp == prevEStack[i+1] { - prevAlt = j - break - } - } - if prevAlt == -1 { - panic("lost alternatives in choiceExpr") - } - return currentAlt < prevAlt, true - } - - return false, false -} - -func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { - result, ok := p.getMemoized(rule) - if ok { - checkPriority, haveChoice := p.checkPrevChoice(rule) - if haveChoice && !checkPriority { - return nil, false - } - p.restore(result.end) - return result.v, result.b - } - - if p.debug { - defer p.out(p.in("recursive " + rule.name)) - } - - var ( - depth = 0 - startMark = p.pt - lastResult = resultTuple{nil, false, startMark} - ) - - for { - lastState := p.cloneState() - p.setMemoized(startMark, rule, lastResult) - val, ok := p.parseRule(rule) - endMark := p.pt - if p.debug { - p.printIndent("RECURSIVE", fmt.Sprintf( - "Rule %s depth %d: %t -> %s", - rule.name, depth, ok, string(p.sliceFrom(startMark)))) - } - if (!ok) || (endMark.offset <= lastResult.end.offset) { - p.restoreState(lastState) - break - } - lastResult = resultTuple{val, ok, endMark} - p.restore(startMark) - depth++ - } - - p.restore(lastResult.end) - p.setMemoized(startMark, rule, lastResult) - return lastResult.v, lastResult.b -} - -func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { - return p.parseRule(rule) -} - func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { res, ok := p.getMemoized(rule) if ok { @@ -1267,14 +1164,8 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { startMark = p.pt ) - if p.memoize || rule.leftRecursive { - if rule.leader { - val, ok = p.parseRuleRecursiveLeader(rule) - } else if p.memoize && !rule.leftRecursive { - val, ok = p.parseRuleMemoize(rule) - } else { - val, ok = p.parseRuleRecursiveNoLeader(rule) - } + if p.memoize { + val, ok = p.parseRuleMemoize(rule) } else { val, ok = p.parseRule(rule) } @@ -1286,11 +1177,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1321,7 +1212,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { panic(errMaxExprCnt) } - p.pushExpr(expr) var val any var ok bool switch expr := expr.(type) { @@ -1364,7 +1254,6 @@ func (p *parser) parseExpr(expr any) (any, bool) { default: panic(fmt.Sprintf("unknown expression type %T", expr)) } - p.popExpr() return val, ok } @@ -1509,7 +1398,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/left_recursion/without_left_recursion.peg b/test/left_recursion/without_left_recursion.peg new file mode 100644 index 00000000..8758cda4 --- /dev/null +++ b/test/left_recursion/without_left_recursion.peg @@ -0,0 +1,42 @@ +{ + package withoutleftrecursion + + func toAnySlice(v any) []any { + if v == nil { + return nil + } + return v.([]any) + } + + func exprToString(first string, rest any) string { + restSl := toAnySlice(rest) + l := first + for _, v := range restSl { + restExpr := toAnySlice(v) + r := restExpr[1].(string) + op := string(restExpr[0].([]byte)) + l = "(" + l + op + r + ")" + } + return l + } +} + +start = a:expr !. { + return a, nil +} +expr = a:term b:(( '+' / '-' ) term )* { + strA := a.(string) + return exprToString(strA, b), nil +} +term = a:factor b:( ( '*' / '/' / '%') factor )* { + strA := a.(string) + return exprToString(strA, b), nil +} +factor = op:('+' / '-') a:factor { + strA := a.(string) + strOp := string(op.([]byte)) + return "(" + strOp + strA + ")", nil +} / atom { + return string(c.text), nil +} +atom = [0-9]+ diff --git a/test/linear/linear.go b/test/linear/linear.go index af2efe89..4c11c271 100644 --- a/test/linear/linear.go +++ b/test/linear/linear.go @@ -740,18 +740,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -819,7 +807,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1079,11 +1067,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1300,7 +1288,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/max_expr_cnt/maxexpr.go b/test/max_expr_cnt/maxexpr.go index 834118ed..29d7938a 100644 --- a/test/max_expr_cnt/maxexpr.go +++ b/test/max_expr_cnt/maxexpr.go @@ -617,18 +617,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -696,7 +684,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -956,11 +944,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1177,7 +1165,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/predicates/predicates.go b/test/predicates/predicates.go index de4f3e7f..093fafd5 100644 --- a/test/predicates/predicates.go +++ b/test/predicates/predicates.go @@ -794,18 +794,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -873,7 +861,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1133,11 +1121,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1354,7 +1342,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/runeerror/runeerror.go b/test/runeerror/runeerror.go index b46bc6e0..11863640 100644 --- a/test/runeerror/runeerror.go +++ b/test/runeerror/runeerror.go @@ -639,18 +639,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -718,7 +706,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -978,11 +966,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1199,7 +1187,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/state/state.go b/test/state/state.go index ee6f3a71..b0ae3917 100644 --- a/test/state/state.go +++ b/test/state/state.go @@ -722,18 +722,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -801,7 +789,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1061,11 +1049,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1282,7 +1270,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/stateclone/stateclone.go b/test/stateclone/stateclone.go index 53380c3e..02171b6c 100644 --- a/test/stateclone/stateclone.go +++ b/test/stateclone/stateclone.go @@ -812,18 +812,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -891,7 +879,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1151,11 +1139,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1372,7 +1360,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/statereadonly/statereadonly.go b/test/statereadonly/statereadonly.go index 92455337..137cd3a0 100644 --- a/test/statereadonly/statereadonly.go +++ b/test/statereadonly/statereadonly.go @@ -793,18 +793,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -872,7 +860,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1132,11 +1120,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1353,7 +1341,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/optimized/staterestore.go b/test/staterestore/optimized/staterestore.go index 4adc27c7..dca9c597 100644 --- a/test/staterestore/optimized/staterestore.go +++ b/test/staterestore/optimized/staterestore.go @@ -1162,18 +1162,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1216,7 +1204,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1415,11 +1403,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } diff --git a/test/staterestore/standard/staterestore.go b/test/staterestore/standard/staterestore.go index 79c8761c..300a4eff 100644 --- a/test/staterestore/standard/staterestore.go +++ b/test/staterestore/standard/staterestore.go @@ -892,18 +892,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -971,7 +959,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1231,11 +1219,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1452,7 +1440,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/staterestore/staterestore.go b/test/staterestore/staterestore.go index 79c8761c..300a4eff 100644 --- a/test/staterestore/staterestore.go +++ b/test/staterestore/staterestore.go @@ -892,18 +892,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -971,7 +959,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -1231,11 +1219,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -1452,7 +1440,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/test/thrownrecover/thrownrecover.go b/test/thrownrecover/thrownrecover.go index ed6914bb..faf0788e 100644 --- a/test/thrownrecover/thrownrecover.go +++ b/test/thrownrecover/thrownrecover.go @@ -1674,18 +1674,6 @@ func (p *parser) popV() { p.vstack = p.vstack[:len(p.vstack)-1] } -func (p *parser) pushRule(rule *rule) { - p.rstack = append(p.rstack, rule) -} - -func (p *parser) popRule() { - p.rstack = p.rstack[:len(p.rstack)-1] -} - -func (p *parser) getRule() *rule { - return p.rstack[len(p.rstack)-1] -} - // push a recovery expression with its labels to the recoveryStack func (p *parser) pushRecovery(labels []string, expr any) { if cap(p.recoveryStack) == len(p.recoveryStack) { @@ -1753,7 +1741,7 @@ func (p *parser) addErrAt(err error, pos position, expected []string) { if buf.Len() > 0 { buf.WriteString(": ") } - rule := p.getRule() + rule := p.rstack[len(p.rstack)-1] if rule.displayName != "" { buf.WriteString("rule " + rule.displayName) } else { @@ -2013,11 +2001,11 @@ func (p *parser) parseRuleWrap(rule *rule) (any, bool) { } func (p *parser) parseRule(rule *rule) (any, bool) { - p.pushRule(rule) + p.rstack = append(p.rstack, rule) p.pushV() val, ok := p.parseExprWrap(rule.expr) p.popV() - p.popRule() + p.rstack = p.rstack[:len(p.rstack)-1] return val, ok } @@ -2234,7 +2222,7 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { } func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.getRule().name, ch.pos.line, ch.pos.col) + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) m := p.ChoiceAltCnt[choiceIdent] if m == nil { m = make(map[string]int) diff --git a/testutils/testutils.go b/testutils/testutils.go index 0b94b117..9a6c3b7d 100644 --- a/testutils/testutils.go +++ b/testutils/testutils.go @@ -1,18 +1,19 @@ +// Copied from https://github.com/stretchr/testify + +// Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. All rights reserved. +// Use of this source code is governed by an MIT-style license that can be found in +// the THIRD-PARTY-NOTICES file. + package testutils import ( "bytes" + "errors" "reflect" ) -// Copied from https://github.com/stretchr/testify - -// Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. All rights reserved. -// Use of this source code is governed by an MIT-style license that can be found in -// the THIRD-PARTY-NOTICES file. - -// isEmpty gets whether the specified object is considered empty or not. -func isEmpty(object interface{}) bool { +// IsEmpty gets whether the specified object is considered empty or not. +func IsEmpty(object interface{}) bool { // get nil case out of the way if object == nil { return true @@ -30,7 +31,7 @@ func isEmpty(object interface{}) bool { return true } deref := objValue.Elem().Interface() - return isEmpty(deref) + return IsEmpty(deref) // for all other types, compare against the zero value // array types are empty when they match their zero-initialized state default: @@ -39,16 +40,16 @@ func isEmpty(object interface{}) bool { } } -// isList checks that the provided value is array or slice. -func isList(list interface{}) (ok bool) { +// IsList checks that the provided value is array or slice. +func IsList(list interface{}) (ok bool) { kind := reflect.TypeOf(list).Kind() return kind == reflect.Array || kind == reflect.Slice } -// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. +// DiffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. // If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and // 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. -func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { +func DiffLists(listA, listB interface{}) (extraA, extraB []interface{}) { aValue := reflect.ValueOf(listA) bValue := reflect.ValueOf(listB) @@ -114,15 +115,50 @@ func ObjectsAreEqual(expected, actual interface{}) bool { // // ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]). func ElementsMatch(listA interface{}, listB interface{}) bool { - if isEmpty(listA) && isEmpty(listB) { + if IsEmpty(listA) && IsEmpty(listB) { return true } - if !isList(listA) || !isList(listB) { + if !IsList(listA) || !IsList(listB) { return false } - extraA, extraB := diffLists(listA, listB) + extraA, extraB := DiffLists(listA, listB) return len(extraA) == 0 && len(extraB) == 0 } + +func isFunction(arg interface{}) bool { + if arg == nil { + return false + } + return reflect.TypeOf(arg).Kind() == reflect.Func +} + +// ValidateEqualArgs checks whether provided arguments can be safely used in the +// Equal/NotEqual functions. +func ValidateEqualArgs(expected, actual interface{}) error { + if expected == nil && actual == nil { + return nil + } + + if isFunction(expected) || isFunction(actual) { + return errors.New("cannot take func type as argument") + } + return nil +} + +// Equal asserts that two objects are equal. +// +// Equal(123, 123) +// +// Pointer variable equality is determined based on the equality of the +// referenced values (as opposed to the memory addresses). Function equality +// cannot be determined and will always fail. +func Equal(expected, actual interface{}, msgAndArgs ...interface{}) bool { + if err := ValidateEqualArgs(expected, actual); err != nil { + return false + } + + return ObjectsAreEqual(expected, actual) +} diff --git a/testutils/testutils_test.go b/testutils/testutils_test.go new file mode 100644 index 00000000..5a41b1ec --- /dev/null +++ b/testutils/testutils_test.go @@ -0,0 +1,303 @@ +// Copied from https://github.com/stretchr/testify + +// Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. All rights reserved. +// Use of this source code is governed by an MIT-style license that can be found in +// the THIRD-PARTY-NOTICES file. + +package testutils_test + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/mna/pigeon/testutils" +) + +func TestIsEmpty(t *testing.T) { + t.Parallel() + + chWithValue := make(chan struct{}, 1) + chWithValue <- struct{}{} + + tests := []struct { + obj interface{} + want bool + }{ + {obj: "", want: true}, + {obj: nil, want: true}, + {obj: []string{}, want: true}, + {obj: 0, want: true}, + {obj: int32(0), want: true}, + {obj: int64(0), want: true}, + {obj: false, want: true}, + {obj: map[string]string{}, want: true}, + {obj: new(time.Time), want: true}, + {obj: time.Time{}, want: true}, + {obj: make(chan struct{}), want: true}, + {obj: [1]int{}, want: true}, + {obj: "something", want: false}, + {obj: errors.New("something"), want: false}, + {obj: []string{"something"}, want: false}, + {obj: 1, want: false}, + {obj: true, want: false}, + {obj: map[string]string{"Hello": "World"}, want: false}, + {obj: chWithValue, want: false}, + {obj: [1]int{42}, want: false}, + } + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("IsEmpty(%#v)", test.obj), func(t *testing.T) { + t.Parallel() + + isEmpty := testutils.IsEmpty(test.obj) + if isEmpty != test.want { + t.Fatalf("IsEmpty(%#v) should return %v", test.obj, test.want) + } + }) + } +} + +func TestVlidateEqualArgs(t *testing.T) { + t.Parallel() + + if testutils.ValidateEqualArgs(func() {}, func() {}) == nil { + t.Error("non-nil functions should error") + } + + if testutils.ValidateEqualArgs(func() {}, func() {}) == nil { + t.Error("non-nil functions should error") + } + + if testutils.ValidateEqualArgs(nil, nil) != nil { + t.Error("nil functions are equal") + } +} + +func TestEqual(t *testing.T) { + t.Parallel() + + type myType string + + var m map[string]interface{} + + tests := []struct { + expected interface{} + actual interface{} + result bool + remark string + }{ + {"Hello World", "Hello World", true, ""}, + {123, 123, true, ""}, + {123.5, 123.5, true, ""}, + {[]byte("Hello World"), []byte("Hello World"), true, ""}, + {nil, nil, true, ""}, + {int32(123), int32(123), true, ""}, + {uint64(123), uint64(123), true, ""}, + {myType("1"), myType("1"), true, ""}, + {&struct{}{}, &struct{}{}, true, "pointer equality is based on equality of underlying value"}, + + // Not expected to be equal + {m["bar"], "something", false, ""}, + {myType("1"), myType("2"), false, ""}, + + // A case that might be confusing, especially with numeric literals + {10, uint(10), false, ""}, + } + + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("Equal(%#v, %#v)", test.expected, test.actual), func(t *testing.T) { + t.Parallel() + + res := testutils.Equal(test.expected, test.actual) + if res != test.result { + t.Errorf( + "Equal(%#v, %#v) should return %#v: %s", + test.expected, test.actual, test.result, test.remark) + } + }) + } +} + +func TestDiffLists(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + listA interface{} + listB interface{} + extraA []interface{} + extraB []interface{} + }{ + { + name: "equal empty", + listA: []string{}, + listB: []string{}, + extraA: nil, + extraB: nil, + }, + { + name: "equal same order", + listA: []string{"hello", "world"}, + listB: []string{"hello", "world"}, + extraA: nil, + extraB: nil, + }, + { + name: "equal different order", + listA: []string{"hello", "world"}, + listB: []string{"world", "hello"}, + extraA: nil, + extraB: nil, + }, + { + name: "extra A", + listA: []string{"hello", "hello", "world"}, + listB: []string{"hello", "world"}, + extraA: []interface{}{"hello"}, + extraB: nil, + }, + { + name: "extra A twice", + listA: []string{"hello", "hello", "hello", "world"}, + listB: []string{"hello", "world"}, + extraA: []interface{}{"hello", "hello"}, + extraB: nil, + }, + { + name: "extra B", + listA: []string{"hello", "world"}, + listB: []string{"hello", "hello", "world"}, + extraA: nil, + extraB: []interface{}{"hello"}, + }, + { + name: "extra B twice", + listA: []string{"hello", "world"}, + listB: []string{"hello", "hello", "world", "hello"}, + extraA: nil, + extraB: []interface{}{"hello", "hello"}, + }, + { + name: "integers 1", + listA: []int{1, 2, 3, 4, 5}, + listB: []int{5, 4, 3, 2, 1}, + extraA: nil, + extraB: nil, + }, + { + name: "integers 2", + listA: []int{1, 2, 1, 2, 1}, + listB: []int{2, 1, 2, 1, 2}, + extraA: []interface{}{1}, + extraB: []interface{}{2}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + actualExtraA, actualExtraB := testutils.DiffLists( + test.listA, test.listB) + if !testutils.Equal(test.extraA, actualExtraA) { + t.Errorf( + "extra A does not match for listA=%v listB=%v", + test.listA, test.listB) + } + if !testutils.Equal(test.extraB, actualExtraB) { + t.Errorf( + "extra B does not match for listA=%v listB=%v", + test.listA, test.listB) + } + }) + } +} + +func TestObjectsAreEqual(t *testing.T) { + t.Parallel() + + cases := []struct { + expected interface{} + actual interface{} + result bool + }{ + // cases that are expected to be equal + {"Hello World", "Hello World", true}, + {123, 123, true}, + {123.5, 123.5, true}, + {[]byte("Hello World"), []byte("Hello World"), true}, + {nil, nil, true}, + + // cases that are expected not to be equal + {map[int]int{5: 10}, map[int]int{10: 20}, false}, + {'x', "x", false}, + {"x", 'x', false}, + {0, 0.1, false}, + {0.1, 0, false}, + {time.Now, time.Now, false}, + {func() {}, func() {}, false}, + {uint32(10), int32(10), false}, + } + + for _, test := range cases { + test := test + t.Run(fmt.Sprintf("ObjectsAreEqual(%#v, %#v)", test.expected, test.actual), func(t *testing.T) { + t.Parallel() + + res := testutils.ObjectsAreEqual(test.expected, test.actual) + if res != test.result { + t.Errorf( + "ObjectsAreEqual(%#v, %#v) should return %#v", + test.expected, test.actual, test.result) + } + }) + } +} + +func TestElementsMatch(t *testing.T) { + t.Parallel() + + tests := []struct { + expected interface{} + actual interface{} + result bool + }{ + // matching + {nil, nil, true}, + + {nil, nil, true}, + {[]int{}, []int{}, true}, + {[]int{1}, []int{1}, true}, + {[]int{1, 1}, []int{1, 1}, true}, + {[]int{1, 2}, []int{1, 2}, true}, + {[]int{1, 2}, []int{2, 1}, true}, + {[2]int{1, 2}, [2]int{2, 1}, true}, + {[]string{"hello", "world"}, []string{"world", "hello"}, true}, + {[]string{"hello", "hello"}, []string{"hello", "hello"}, true}, + {[]string{"hello", "hello", "world"}, []string{"hello", "world", "hello"}, true}, + {[3]string{"hello", "hello", "world"}, [3]string{"hello", "world", "hello"}, true}, + {[]int{}, nil, true}, + + // not matching + {[]int{1}, []int{1, 1}, false}, + {[]int{1, 2}, []int{2, 2}, false}, + {[]string{"hello", "hello"}, []string{"hello"}, false}, + } + + for _, test := range tests { + test := test + t.Run(fmt.Sprintf("ElementsMatch(%#v, %#v)", test.expected, test.actual), func(t *testing.T) { + t.Parallel() + + res := testutils.ElementsMatch(test.actual, test.expected) + if res != test.result { + t.Errorf( + "ElementsMatch(%#v, %#v) should return %v", + test.actual, test.expected, test.result) + } + }) + } +} From f39183d3c638b1491769e650a04bcd937266d141 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Wed, 5 Jul 2023 20:04:07 +0300 Subject: [PATCH 09/14] Fix formatting --- test/left_recursion/left_recursion.peg | 4 ++-- test/left_recursion/left_recursion_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg index 792638fa..895b521c 100644 --- a/test/left_recursion/left_recursion.peg +++ b/test/left_recursion/left_recursion.peg @@ -6,7 +6,7 @@ start = a:expr !. { return a, nil } -expr = a:expr op:('+'/'-') b:term { +expr = a:expr op:('+' / '-') b:term { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) @@ -16,7 +16,7 @@ expr = a:expr op:('+'/'-') b:term { return strA, nil } -term = a:term op:('*'/'/'/'%') b:factor { +term = a:term op:('*' / '/' / '%') b:factor { strA := a.(string) strB := b.(string) strOp := string(op.([]byte)) diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/left_recursion_test.go index aa31c22b..81885737 100644 --- a/test/left_recursion/left_recursion_test.go +++ b/test/left_recursion/left_recursion_test.go @@ -91,7 +91,7 @@ func TestLeftRecursionParse(t *testing.T) { } }) - t.Run(testCase.name+" optimezed", func(t *testing.T) { + t.Run(testCase.name+" optimized", func(t *testing.T) { t.Parallel() resLR, err := optimizedleftrecursion.Parse("", []byte(testCase.expr)) From f458d765d22ff8695dbe72ec8e843b8564f5d53e Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Thu, 6 Jul 2023 14:30:31 +0300 Subject: [PATCH 10/14] Add left recursion tests with state, labeled failures, throw and recover --- Makefile | 22 + builder/generated_static_code.go | 5 +- builder/static_code.go | 5 +- test/issue_70b/issue_70b.go | 5 +- .../optimized/leftrecursion/left_recursion.go | 79 +- .../standart/leftrecursion/left_recursion.go | 79 +- .../left_recursion_labeled_failures/errors.go | 31 + .../left_recursion_labeled_failures.go | 1754 +++++++++++ .../left_recursion_labeled_failures.peg | 40 + .../left_recursion_labeled_failures_test.go | 154 + .../left_recursion_state.peg | 41 + .../left_recursion_state_test.go | 103 + .../optimized/left_recursion_state.go | 1533 ++++++++++ .../standart/left_recursion_state.go | 1764 +++++++++++ test/left_recursion_thrownrecover/errors.go | 31 + .../left_recursion_thrownrecover.go | 2587 +++++++++++++++++ .../left_recursion_thrownrecover.peg | 106 + .../left_recursion_thrownrecover_test.go | 204 ++ 18 files changed, 8464 insertions(+), 79 deletions(-) create mode 100644 test/left_recursion_labeled_failures/errors.go create mode 100644 test/left_recursion_labeled_failures/left_recursion_labeled_failures.go create mode 100644 test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg create mode 100644 test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go create mode 100644 test/left_recursion_state/left_recursion_state.peg create mode 100644 test/left_recursion_state/left_recursion_state_test.go create mode 100644 test/left_recursion_state/optimized/left_recursion_state.go create mode 100644 test/left_recursion_state/standart/left_recursion_state.go create mode 100644 test/left_recursion_thrownrecover/errors.go create mode 100644 test/left_recursion_thrownrecover/left_recursion_thrownrecover.go create mode 100644 test/left_recursion_thrownrecover/left_recursion_thrownrecover.peg create mode 100644 test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go diff --git a/Makefile b/Makefile index c0a8c914..932d3281 100644 --- a/Makefile +++ b/Makefile @@ -200,6 +200,28 @@ $(TEST_DIR)/left_recursion/optimized/withoutleftrecursion/without_left_recursion $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint -optimize-parser $< > $@ +$(TEST_DIR)/left_recursion_state/left_recursion_state.go: \ + $(TEST_DIR)/left_recursion_state/standart/left_recursion_state.go \ + $(TEST_DIR)/left_recursion_state/optimized/left_recursion_state.go \ + $(BINDIR)/pigeon + +$(TEST_DIR)/left_recursion_state/standart/left_recursion_state.go: \ + $(TEST_DIR)/left_recursion_state/left_recursion_state.peg $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ + +$(TEST_DIR)/left_recursion_state/optimized/left_recursion_state.go: \ + $(TEST_DIR)/left_recursion_state/left_recursion_state.peg $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -optimize-parser -support-left-recursion $< > $@ + +$(TEST_DIR)/left_recursion_labeled_failures/left_recursion_labeled_failures.go: \ + $(TEST_DIR)/left_recursion_labeled_failures/left_recursion_labeled_failures.peg \ + $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ + +$(TEST_DIR)/left_recursion_thrownrecover/left_recursion_thrownrecover.go: \ + $(TEST_DIR)/left_recursion_thrownrecover/left_recursion_thrownrecover.peg \ + $(BINDIR)/pigeon + $(BINDIR)/pigeon -nolint -support-left-recursion $< > $@ lint: golint ./... diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index 435018c0..ca61a9fd 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -909,6 +909,7 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { depth = 0 startMark = p.pt lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs ) for { @@ -925,13 +926,15 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { rule.name, depth, ok, string(p.sliceFrom(startMark)))) } // {{ end }} ==template== - if (!ok) || (endMark.offset <= lastResult.end.offset) { + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(lastState) // {{ end }} ==template== + *p.errs = lastErrors break } lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs p.restore(startMark) depth++ } diff --git a/builder/static_code.go b/builder/static_code.go index c4df7c17..2f5b4d62 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -927,6 +927,7 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { depth = 0 startMark = p.pt lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs ) for { @@ -943,13 +944,15 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { rule.name, depth, ok, string(p.sliceFrom(startMark)))) } // {{ end }} ==template== - if (!ok) || (endMark.offset <= lastResult.end.offset) { + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { // ==template== {{ if or .GlobalState (not .Optimize) }} p.restoreState(lastState) // {{ end }} ==template== + *p.errs = lastErrors break } lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs p.restore(startMark) depth++ } diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index 730bd0ef..cb5b2c91 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -941,6 +941,7 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { depth = 0 startMark = p.pt lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs ) for { @@ -953,11 +954,13 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { "Rule %s depth %d: %t -> %s", rule.name, depth, ok, string(p.sliceFrom(startMark)))) } - if (!ok) || (endMark.offset <= lastResult.end.offset) { + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { p.restoreState(lastState) + *p.errs = lastErrors break } lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs p.restore(startMark) depth++ } diff --git a/test/left_recursion/optimized/leftrecursion/left_recursion.go b/test/left_recursion/optimized/leftrecursion/left_recursion.go index 6a373f4e..b61a0fd2 100644 --- a/test/left_recursion/optimized/leftrecursion/left_recursion.go +++ b/test/left_recursion/optimized/leftrecursion/left_recursion.go @@ -80,7 +80,7 @@ var g = &grammar{ want: "\"+\"", }, &litMatcher{ - pos: position{line: 9, col: 24, offset: 90}, + pos: position{line: 9, col: 26, offset: 92}, val: "-", ignoreCase: false, want: "\"-\"", @@ -89,10 +89,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 9, col: 29, offset: 95}, + pos: position{line: 9, col: 31, offset: 97}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 9, col: 31, offset: 97}, + pos: position{line: 9, col: 33, offset: 99}, name: "term", }, }, @@ -100,13 +100,13 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 14, col: 5, offset: 235}, + pos: position{line: 14, col: 5, offset: 237}, run: (*parser).callonexpr12, expr: &labeledExpr{ - pos: position{line: 14, col: 5, offset: 235}, + pos: position{line: 14, col: 5, offset: 237}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 14, col: 7, offset: 237}, + pos: position{line: 14, col: 7, offset: 239}, name: "term", }, }, @@ -118,44 +118,44 @@ var g = &grammar{ }, { name: "term", - pos: position{line: 19, col: 1, offset: 291}, + pos: position{line: 19, col: 1, offset: 293}, expr: &choiceExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, alternatives: []any{ &actionExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, run: (*parser).callonterm2, expr: &seqExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, exprs: []any{ &labeledExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 19, col: 10, offset: 300}, + pos: position{line: 19, col: 10, offset: 302}, name: "term", }, }, &labeledExpr{ - pos: position{line: 19, col: 15, offset: 305}, + pos: position{line: 19, col: 15, offset: 307}, label: "op", expr: &choiceExpr{ - pos: position{line: 19, col: 19, offset: 309}, + pos: position{line: 19, col: 19, offset: 311}, alternatives: []any{ &litMatcher{ - pos: position{line: 19, col: 19, offset: 309}, + pos: position{line: 19, col: 19, offset: 311}, val: "*", ignoreCase: false, want: "\"*\"", }, &litMatcher{ - pos: position{line: 19, col: 23, offset: 313}, + pos: position{line: 19, col: 25, offset: 317}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 19, col: 27, offset: 317}, + pos: position{line: 19, col: 31, offset: 323}, val: "%", ignoreCase: false, want: "\"%\"", @@ -164,10 +164,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 19, col: 32, offset: 322}, + pos: position{line: 19, col: 36, offset: 328}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 19, col: 34, offset: 324}, + pos: position{line: 19, col: 38, offset: 330}, name: "factor", }, }, @@ -175,13 +175,13 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 25, col: 5, offset: 466}, + pos: position{line: 25, col: 5, offset: 472}, run: (*parser).callonterm13, expr: &labeledExpr{ - pos: position{line: 25, col: 5, offset: 466}, + pos: position{line: 25, col: 5, offset: 472}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 25, col: 7, offset: 468}, + pos: position{line: 25, col: 7, offset: 474}, name: "factor", }, }, @@ -193,30 +193,30 @@ var g = &grammar{ }, { name: "factor", - pos: position{line: 30, col: 1, offset: 524}, + pos: position{line: 30, col: 1, offset: 530}, expr: &choiceExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, alternatives: []any{ &actionExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, run: (*parser).callonfactor2, expr: &seqExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, exprs: []any{ &labeledExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, label: "op", expr: &choiceExpr{ - pos: position{line: 30, col: 14, offset: 537}, + pos: position{line: 30, col: 14, offset: 543}, alternatives: []any{ &litMatcher{ - pos: position{line: 30, col: 14, offset: 537}, + pos: position{line: 30, col: 14, offset: 543}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 30, col: 20, offset: 543}, + pos: position{line: 30, col: 20, offset: 549}, val: "-", ignoreCase: false, want: "\"-\"", @@ -225,10 +225,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 30, col: 25, offset: 548}, + pos: position{line: 30, col: 25, offset: 554}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 30, col: 27, offset: 550}, + pos: position{line: 30, col: 27, offset: 556}, name: "factor", }, }, @@ -236,10 +236,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 34, col: 5, offset: 660}, + pos: position{line: 34, col: 5, offset: 666}, run: (*parser).callonfactor10, expr: &ruleRefExpr{ - pos: position{line: 34, col: 5, offset: 660}, + pos: position{line: 34, col: 5, offset: 666}, name: "atom", }, }, @@ -250,11 +250,11 @@ var g = &grammar{ }, { name: "atom", - pos: position{line: 38, col: 1, offset: 701}, + pos: position{line: 38, col: 1, offset: 707}, expr: &oneOrMoreExpr{ - pos: position{line: 38, col: 8, offset: 708}, + pos: position{line: 38, col: 8, offset: 714}, expr: &charClassMatcher{ - pos: position{line: 38, col: 8, offset: 708}, + pos: position{line: 38, col: 8, offset: 714}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1041,16 +1041,19 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { depth = 0 startMark = p.pt lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs ) for { p.setMemoized(startMark, rule, lastResult) val, ok := p.parseRule(rule) endMark := p.pt - if (!ok) || (endMark.offset <= lastResult.end.offset) { + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + *p.errs = lastErrors break } lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs p.restore(startMark) depth++ } diff --git a/test/left_recursion/standart/leftrecursion/left_recursion.go b/test/left_recursion/standart/leftrecursion/left_recursion.go index 4c8a256b..d72b8bed 100644 --- a/test/left_recursion/standart/leftrecursion/left_recursion.go +++ b/test/left_recursion/standart/leftrecursion/left_recursion.go @@ -81,7 +81,7 @@ var g = &grammar{ want: "\"+\"", }, &litMatcher{ - pos: position{line: 9, col: 24, offset: 90}, + pos: position{line: 9, col: 26, offset: 92}, val: "-", ignoreCase: false, want: "\"-\"", @@ -90,10 +90,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 9, col: 29, offset: 95}, + pos: position{line: 9, col: 31, offset: 97}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 9, col: 31, offset: 97}, + pos: position{line: 9, col: 33, offset: 99}, name: "term", }, }, @@ -101,13 +101,13 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 14, col: 5, offset: 235}, + pos: position{line: 14, col: 5, offset: 237}, run: (*parser).callonexpr12, expr: &labeledExpr{ - pos: position{line: 14, col: 5, offset: 235}, + pos: position{line: 14, col: 5, offset: 237}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 14, col: 7, offset: 237}, + pos: position{line: 14, col: 7, offset: 239}, name: "term", }, }, @@ -119,44 +119,44 @@ var g = &grammar{ }, { name: "term", - pos: position{line: 19, col: 1, offset: 291}, + pos: position{line: 19, col: 1, offset: 293}, expr: &choiceExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, alternatives: []any{ &actionExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, run: (*parser).callonterm2, expr: &seqExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, exprs: []any{ &labeledExpr{ - pos: position{line: 19, col: 8, offset: 298}, + pos: position{line: 19, col: 8, offset: 300}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 19, col: 10, offset: 300}, + pos: position{line: 19, col: 10, offset: 302}, name: "term", }, }, &labeledExpr{ - pos: position{line: 19, col: 15, offset: 305}, + pos: position{line: 19, col: 15, offset: 307}, label: "op", expr: &choiceExpr{ - pos: position{line: 19, col: 19, offset: 309}, + pos: position{line: 19, col: 19, offset: 311}, alternatives: []any{ &litMatcher{ - pos: position{line: 19, col: 19, offset: 309}, + pos: position{line: 19, col: 19, offset: 311}, val: "*", ignoreCase: false, want: "\"*\"", }, &litMatcher{ - pos: position{line: 19, col: 23, offset: 313}, + pos: position{line: 19, col: 25, offset: 317}, val: "/", ignoreCase: false, want: "\"/\"", }, &litMatcher{ - pos: position{line: 19, col: 27, offset: 317}, + pos: position{line: 19, col: 31, offset: 323}, val: "%", ignoreCase: false, want: "\"%\"", @@ -165,10 +165,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 19, col: 32, offset: 322}, + pos: position{line: 19, col: 36, offset: 328}, label: "b", expr: &ruleRefExpr{ - pos: position{line: 19, col: 34, offset: 324}, + pos: position{line: 19, col: 38, offset: 330}, name: "factor", }, }, @@ -176,13 +176,13 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 25, col: 5, offset: 466}, + pos: position{line: 25, col: 5, offset: 472}, run: (*parser).callonterm13, expr: &labeledExpr{ - pos: position{line: 25, col: 5, offset: 466}, + pos: position{line: 25, col: 5, offset: 472}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 25, col: 7, offset: 468}, + pos: position{line: 25, col: 7, offset: 474}, name: "factor", }, }, @@ -194,30 +194,30 @@ var g = &grammar{ }, { name: "factor", - pos: position{line: 30, col: 1, offset: 524}, + pos: position{line: 30, col: 1, offset: 530}, expr: &choiceExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, alternatives: []any{ &actionExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, run: (*parser).callonfactor2, expr: &seqExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, exprs: []any{ &labeledExpr{ - pos: position{line: 30, col: 10, offset: 533}, + pos: position{line: 30, col: 10, offset: 539}, label: "op", expr: &choiceExpr{ - pos: position{line: 30, col: 14, offset: 537}, + pos: position{line: 30, col: 14, offset: 543}, alternatives: []any{ &litMatcher{ - pos: position{line: 30, col: 14, offset: 537}, + pos: position{line: 30, col: 14, offset: 543}, val: "+", ignoreCase: false, want: "\"+\"", }, &litMatcher{ - pos: position{line: 30, col: 20, offset: 543}, + pos: position{line: 30, col: 20, offset: 549}, val: "-", ignoreCase: false, want: "\"-\"", @@ -226,10 +226,10 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 30, col: 25, offset: 548}, + pos: position{line: 30, col: 25, offset: 554}, label: "a", expr: &ruleRefExpr{ - pos: position{line: 30, col: 27, offset: 550}, + pos: position{line: 30, col: 27, offset: 556}, name: "factor", }, }, @@ -237,10 +237,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 34, col: 5, offset: 660}, + pos: position{line: 34, col: 5, offset: 666}, run: (*parser).callonfactor10, expr: &ruleRefExpr{ - pos: position{line: 34, col: 5, offset: 660}, + pos: position{line: 34, col: 5, offset: 666}, name: "atom", }, }, @@ -251,11 +251,11 @@ var g = &grammar{ }, { name: "atom", - pos: position{line: 38, col: 1, offset: 701}, + pos: position{line: 38, col: 1, offset: 707}, expr: &oneOrMoreExpr{ - pos: position{line: 38, col: 8, offset: 708}, + pos: position{line: 38, col: 8, offset: 714}, expr: &charClassMatcher{ - pos: position{line: 38, col: 8, offset: 708}, + pos: position{line: 38, col: 8, offset: 714}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1207,6 +1207,7 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { depth = 0 startMark = p.pt lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs ) for { @@ -1219,11 +1220,13 @@ func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { "Rule %s depth %d: %t -> %s", rule.name, depth, ok, string(p.sliceFrom(startMark)))) } - if (!ok) || (endMark.offset <= lastResult.end.offset) { + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { p.restoreState(lastState) + *p.errs = lastErrors break } lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs p.restore(startMark) depth++ } diff --git a/test/left_recursion_labeled_failures/errors.go b/test/left_recursion_labeled_failures/errors.go new file mode 100644 index 00000000..f3dca039 --- /dev/null +++ b/test/left_recursion_labeled_failures/errors.go @@ -0,0 +1,31 @@ +package leftrecursionlabeledfailures + +// ErrorLister is the public interface to access the inner errors +// included in a errList. +type ErrorLister interface { + Errors() []error +} + +func (e errList) Errors() []error { + return e +} + +// ParserError is the public interface to errors of type parserError. +type ParserError interface { + Error() string + InnerError() error + Pos() (int, int, int) + Expected() []string +} + +func (p *parserError) InnerError() error { + return p.Inner +} + +func (p *parserError) Pos() (line, col, offset int) { + return p.pos.line, p.pos.col, p.pos.offset +} + +func (p *parserError) Expected() []string { + return p.expected +} diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go new file mode 100644 index 00000000..05330634 --- /dev/null +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go @@ -0,0 +1,1754 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursionlabeledfailures + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +func ids(list, id any) (any, error) { + l := toStringSlice(list) + l = append(l, id.(string)) + return l, nil +} + +func toStringSlice(v any) []string { + if v == nil { + return nil + } + return v.([]string) +} + +var g = &grammar{ + rules: []*rule{ + { + name: "S", + pos: position{line: 18, col: 1, offset: 244}, + expr: &recoveryExpr{ + pos: position{line: 18, col: 5, offset: 250}, + expr: &recoveryExpr{ + pos: position{line: 18, col: 5, offset: 250}, + expr: &actionExpr{ + pos: position{line: 18, col: 5, offset: 250}, + run: (*parser).callonS3, + expr: &labeledExpr{ + pos: position{line: 18, col: 5, offset: 250}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 18, col: 10, offset: 255}, + name: "List", + }, + }, + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 20, col: 16, offset: 309}, + name: "ErrComma", + }, + failureLabel: []string{ + "errComma", + }, + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 20, col: 35, offset: 328}, + name: "ErrID", + }, + failureLabel: []string{ + "errId", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "List", + pos: position{line: 22, col: 1, offset: 335}, + expr: &choiceExpr{ + pos: position{line: 22, col: 8, offset: 344}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 22, col: 8, offset: 344}, + run: (*parser).callonList2, + expr: &seqExpr{ + pos: position{line: 22, col: 9, offset: 345}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 22, col: 9, offset: 345}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 22, col: 14, offset: 350}, + name: "List", + }, + }, + &ruleRefExpr{ + pos: position{line: 22, col: 19, offset: 355}, + name: "Comma", + }, + &labeledExpr{ + pos: position{line: 22, col: 25, offset: 361}, + label: "id", + expr: &ruleRefExpr{ + pos: position{line: 22, col: 28, offset: 364}, + name: "ID", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 24, col: 5, offset: 399}, + run: (*parser).callonList9, + expr: &labeledExpr{ + pos: position{line: 24, col: 5, offset: 399}, + label: "id", + expr: &ruleRefExpr{ + pos: position{line: 24, col: 8, offset: 402}, + name: "ID", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "ID", + pos: position{line: 28, col: 1, offset: 448}, + expr: &choiceExpr{ + pos: position{line: 28, col: 6, offset: 455}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 28, col: 6, offset: 455}, + run: (*parser).callonID2, + expr: &seqExpr{ + pos: position{line: 28, col: 6, offset: 455}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 28, col: 6, offset: 455}, + name: "Sp", + }, + &oneOrMoreExpr{ + pos: position{line: 28, col: 9, offset: 458}, + expr: &charClassMatcher{ + pos: position{line: 28, col: 9, offset: 458}, + val: "[a-z]", + ranges: []rune{'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + &throwExpr{ + pos: position{line: 30, col: 5, offset: 531}, + label: "errId", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "Comma", + pos: position{line: 32, col: 1, offset: 541}, + expr: &choiceExpr{ + pos: position{line: 32, col: 9, offset: 551}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 32, col: 9, offset: 551}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 32, col: 9, offset: 551}, + name: "Sp", + }, + &litMatcher{ + pos: position{line: 32, col: 12, offset: 554}, + val: ",", + ignoreCase: false, + want: "\",\"", + }, + }, + }, + &throwExpr{ + pos: position{line: 32, col: 18, offset: 560}, + label: "errComma", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "Sp", + pos: position{line: 33, col: 1, offset: 572}, + expr: &zeroOrMoreExpr{ + pos: position{line: 33, col: 6, offset: 579}, + expr: &charClassMatcher{ + pos: position{line: 33, col: 6, offset: 579}, + val: "[ \\t\\r\\n]", + chars: []rune{' ', '\t', '\r', '\n'}, + ignoreCase: false, + inverted: false, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrComma", + pos: position{line: 35, col: 1, offset: 591}, + expr: &seqExpr{ + pos: position{line: 35, col: 12, offset: 604}, + exprs: []any{ + &stateCodeExpr{ + pos: position{line: 35, col: 12, offset: 604}, + run: (*parser).callonErrComma2, + }, + &zeroOrMoreExpr{ + pos: position{line: 37, col: 7, offset: 656}, + expr: &seqExpr{ + pos: position{line: 37, col: 9, offset: 658}, + exprs: []any{ + ¬Expr{ + pos: position{line: 37, col: 9, offset: 658}, + expr: &oneOrMoreExpr{ + pos: position{line: 37, col: 11, offset: 660}, + expr: &charClassMatcher{ + pos: position{line: 37, col: 11, offset: 660}, + val: "[a-z]", + ranges: []rune{'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &anyMatcher{ + line: 37, col: 19, offset: 668, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrID", + pos: position{line: 38, col: 1, offset: 672}, + expr: &actionExpr{ + pos: position{line: 38, col: 9, offset: 682}, + run: (*parser).callonErrID1, + expr: &seqExpr{ + pos: position{line: 38, col: 9, offset: 682}, + exprs: []any{ + &stateCodeExpr{ + pos: position{line: 38, col: 9, offset: 682}, + run: (*parser).callonErrID3, + }, + &zeroOrMoreExpr{ + pos: position{line: 40, col: 7, offset: 744}, + expr: &seqExpr{ + pos: position{line: 40, col: 9, offset: 746}, + exprs: []any{ + ¬Expr{ + pos: position{line: 40, col: 9, offset: 746}, + expr: &litMatcher{ + pos: position{line: 40, col: 11, offset: 748}, + val: ",", + ignoreCase: false, + want: "\",\"", + }, + }, + &anyMatcher{ + line: 40, col: 16, offset: 753, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onS3(list any) (any, error) { + return list.([]string), nil +} + +func (p *parser) callonS3() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onS3(stack["list"]) +} + +func (c *current) onList2(list, id any) (any, error) { + return ids(list, id) +} + +func (p *parser) callonList2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onList2(stack["list"], stack["id"]) +} + +func (c *current) onList9(id any) (any, error) { + return []string{id.(string)}, nil +} + +func (p *parser) callonList9() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onList9(stack["id"]) +} + +func (c *current) onID2() (any, error) { + return strings.TrimLeft(string(c.text), " \t\r\n"), nil +} + +func (p *parser) callonID2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onID2() +} + +func (c *current) onErrComma2() error { + return errors.New("expecting ','") + +} + +func (p *parser) callonErrComma2() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrComma2() +} + +func (c *current) onErrID3() error { + return errors.New("expecting an identifier") + +} + +func (p *parser) callonErrID3() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrID3() +} + +func (c *current) onErrID1() (any, error) { + return "NONE", nil +} + +func (p *parser) callonErrID1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrID1() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) + } + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + p.restoreState(lastState) + *p.errs = lastErrors + break + } + lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg new file mode 100644 index 00000000..d0b3181c --- /dev/null +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg @@ -0,0 +1,40 @@ +{ +package leftrecursionlabeledfailures + +func ids(list, id any) (any, error) { + l := toStringSlice(list) + l = append(l, id.(string)) + return l, nil +} + +func toStringSlice(v any) []string { + if v == nil { + return nil + } + return v.([]string) +} +} + +S ← list:List { + return list.([]string), nil +} //{errComma} ErrComma //{errId} ErrID + +List ← (list:List Comma id:ID) { + return ids(list, id) +} / id:ID { + return []string{id.(string)}, nil +} + +ID ← Sp [a-z]+ { + return strings.TrimLeft(string(c.text), " \t\r\n"), nil +} / %{errId} + +Comma ← Sp ',' / %{errComma} +Sp ← [ \t\r\n]* + +ErrComma ← #{ + return errors.New("expecting ','") + } ( !([a-z]+) .)* +ErrID ← #{ + return errors.New("expecting an identifier") + } ( !(',') .)* { return "NONE", nil } \ No newline at end of file diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go new file mode 100644 index 00000000..b5938994 --- /dev/null +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go @@ -0,0 +1,154 @@ +package leftrecursionlabeledfailures_test + +import ( + "reflect" + "testing" + + leftrecursionlabeledfailures "github.com/mna/pigeon/test/left_recursion_labeled_failures" +) + +func TestLeftRecursionWithLabeledFailures(t *testing.T) { + t.Parallel() + + type want struct { + captures []string + errors []string + } + + cases := []struct { + name string + input string + want want + }{ + // Test cases from reference implementation peglabel: + // https://github.com/sqmedeiros/lpeglabel/blob/976b38458e0bba58ca748e96b53afd9ee74a1d1d/README.md#relabel-syntax + // https://github.com/sqmedeiros/lpeglabel/blame/976b38458e0bba58ca748e96b53afd9ee74a1d1d/README.md#L418-L440 + { + name: "correct", + input: "one,two", + want: want{captures: []string{"one", "two"}}, + }, + { + name: "missing commas", + input: "one two three", + want: want{ + captures: []string{"one", "two", "three"}, + errors: []string{ + "1:4 (3): rule ErrComma: expecting ','", + "1:8 (7): rule ErrComma: expecting ','", + }, + }, + }, + { + name: "missing id and incorrect ids", + input: "1,\n two, \n3,", + want: want{ + captures: []string{"NONE", "two", "NONE", "NONE"}, + errors: []string{ + "1:1 (0): rule ErrID: expecting an identifier", + "2:6 (8): rule ErrID: expecting an identifier", + // is line 3, col 2 in peglabel, pigeon increments the position + // behind the last character of the input if !. is matched + "3:3 (12): rule ErrID: expecting an identifier", + }, + }, + }, + { + name: "missing comma, id and incorrect id", + input: "one\n two123, \nthree,", + want: want{ + captures: []string{"one", "two", "three", "NONE"}, + errors: []string{ + // is line 2, col 1 in peglabel, in pigeon, if a \n causes + // an error, this is at col 0 + "2:0 (3): rule ErrComma: expecting ','", + "2:5 (8): rule ErrComma: expecting ','", + // is line 3, col 6 in peglabel, pigeon increments the position + // behind the last character of the input if !. is matched + "3:7 (20): rule ErrID: expecting an identifier", + }, + }, + }, + // Additional test cases + { + name: "empty", + input: "", + want: want{ + captures: []string{"NONE"}, + errors: []string{ + "1:1 (0): rule ErrID: expecting an identifier", + }, + }, + }, + { + name: "incorrect id", + input: "1", + want: want{ + captures: []string{"NONE"}, + errors: []string{"1:1 (0): rule ErrID: expecting an identifier"}, + }, + }, + { + name: "incorrect ids", + input: "1,2", + want: want{ + captures: []string{"NONE", "NONE"}, + errors: []string{ + "1:1 (0): rule ErrID: expecting an identifier", + "1:3 (2): rule ErrID: expecting an identifier", + }, + }, + }, + } + for _, testCase := range cases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + got, err := leftrecursionlabeledfailures.Parse( + "", []byte(testCase.input)) + if testCase.want.errors == nil && err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.input, err) + } + if !reflect.DeepEqual(got, testCase.want.captures) { + t.Errorf( + "for input %q want %s, got %s", + testCase.input, testCase.want.captures, got) + } + if err != nil { + errorLister, ok := err.(leftrecursionlabeledfailures.ErrorLister) + if !ok { + t.FailNow() + } + list := errorLister.Errors() + if len(list) != len(testCase.want.errors) { + t.Errorf( + "for input %q want %d error(s), got %d", + testCase.input, len(testCase.want.errors), len(list)) + t.Logf("expected errors:\n") + for _, ee := range testCase.want.errors { + t.Logf("- %s\n", ee) + } + t.Logf("got errors:\n") + for _, ee := range list { + t.Logf("- %s\n", ee) + } + t.FailNow() + } + for index, err := range list { + pe, ok := err.(leftrecursionlabeledfailures.ParserError) + if !ok { + t.FailNow() + } + if pe.Error() != testCase.want.errors[index] { + t.Errorf( + "for input %q want %dth error to be %s, got %s", + testCase.input, index+1, testCase.want.errors[index], pe) + } + } + } + }) + } +} diff --git a/test/left_recursion_state/left_recursion_state.peg b/test/left_recursion_state/left_recursion_state.peg new file mode 100644 index 00000000..679dc2ce --- /dev/null +++ b/test/left_recursion_state/left_recursion_state.peg @@ -0,0 +1,41 @@ +{ + package leftrecursionstate +} + +start = #{ + if _, ok := c.state["count"]; !ok { + c.state["count"] = 0 + } + return nil +} (a:expr)? { + return c.state["count"], nil +} + +expr = (expr ('+' / '-') term) #{ + c.state["count"] = c.state["count"].(int) + 1; + return nil +} / term #{ + c.state["count"] = c.state["count"].(int) + 3; + return nil +} + +term = (term ('*' / '/' / '%') factor) #{ + c.state["count"] = c.state["count"].(int) + 7; + return nil +} / factor #{ + c.state["count"] = c.state["count"].(int) + 15; + return nil +} + +factor = (('+' / '-') factor) #{ + c.state["count"] = c.state["count"].(int) + 31; + return nil +} / atom #{ + c.state["count"] = c.state["count"].(int) + 63; + return nil +} + +atom = ([0-9]+) #{ + c.state["count"] = c.state["count"].(int) + 127; + return nil +} \ No newline at end of file diff --git a/test/left_recursion_state/left_recursion_state_test.go b/test/left_recursion_state/left_recursion_state_test.go new file mode 100644 index 00000000..db03d37e --- /dev/null +++ b/test/left_recursion_state/left_recursion_state_test.go @@ -0,0 +1,103 @@ +package leftrecursionstate_test + +import ( + "testing" + + optimizedleftrecursionstate "github.com/mna/pigeon/test/left_recursion_state/optimized" + leftrecursionstate "github.com/mna/pigeon/test/left_recursion_state/standart" +) + +func TestLeftRecursionWithState(t *testing.T) { + t.Parallel() + + initCount := 100000 + + type want struct { + count int + } + + tests := []struct { + name string + expr string + want want + }{ + { + name: "atom", + expr: "1", + want: want{count: 3 + 15 + 63 + 127 + initCount}, + }, + { + name: "factor", + expr: "-1", + want: want{count: 3 + 15 + 31 + 63 + 127 + initCount}, + }, + { + name: "expr", + expr: "1+1", + want: want{count: 1 + + (3 + 15 + 63 + 127) + + (15 + 63 + 127) + initCount}, + }, + { + name: "expr", + expr: "1*1*1", + want: want{count: 3 + + 7 + + 7 + + (15 + 63 + 127) + + (63 + 127) + + (63 + 127) + + +initCount}, + }, + { + name: "invalid", + expr: "**", + want: want{count: initCount}, + }, + } + + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name+" default", func(t *testing.T) { + t.Parallel() + + count, err := leftrecursionstate.Parse( + "", []byte(testCase.expr), + leftrecursionstate.Memoize(false), + leftrecursionstate.InitState("count", initCount)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + if count != testCase.want.count { + t.Fatalf( + "for input %q\ngot result: %d,\nbut expect: %d", + testCase.expr, count, testCase.want.count) + } + }) + + t.Run(testCase.name+" optimized", func(t *testing.T) { + t.Parallel() + + count, err := optimizedleftrecursionstate.Parse( + "", []byte(testCase.expr), + optimizedleftrecursionstate.InitState("count", initCount)) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + if count != testCase.want.count { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, count, testCase.want.count) + } + if count != testCase.want.count { + t.Fatalf( + "for input %q\ngot result: %d,\nbut expect: %d", + testCase.expr, count, testCase.want.count) + } + }) + } +} diff --git a/test/left_recursion_state/optimized/left_recursion_state.go b/test/left_recursion_state/optimized/left_recursion_state.go new file mode 100644 index 00000000..e6a9787f --- /dev/null +++ b/test/left_recursion_state/optimized/left_recursion_state.go @@ -0,0 +1,1533 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursionstate + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 5, col: 1, offset: 34}, + expr: &actionExpr{ + pos: position{line: 5, col: 9, offset: 42}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 5, col: 9, offset: 42}, + exprs: []any{ + &stateCodeExpr{ + pos: position{line: 5, col: 9, offset: 42}, + run: (*parser).callonstart3, + }, + &zeroOrOneExpr{ + pos: position{line: 10, col: 3, offset: 128}, + expr: &labeledExpr{ + pos: position{line: 10, col: 4, offset: 129}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 10, col: 6, offset: 131}, + name: "expr", + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "expr", + pos: position{line: 14, col: 1, offset: 173}, + expr: &choiceExpr{ + pos: position{line: 14, col: 9, offset: 181}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 14, col: 9, offset: 181}, + exprs: []any{ + &seqExpr{ + pos: position{line: 14, col: 10, offset: 182}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 14, col: 10, offset: 182}, + name: "expr", + }, + &choiceExpr{ + pos: position{line: 14, col: 16, offset: 188}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 14, col: 16, offset: 188}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 14, col: 22, offset: 194}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 14, col: 27, offset: 199}, + name: "term", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 14, col: 33, offset: 205}, + run: (*parser).callonexpr9, + }, + }, + }, + &seqExpr{ + pos: position{line: 17, col: 5, offset: 278}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 17, col: 5, offset: 278}, + name: "term", + }, + &stateCodeExpr{ + pos: position{line: 17, col: 10, offset: 283}, + run: (*parser).callonexpr12, + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 22, col: 1, offset: 355}, + expr: &choiceExpr{ + pos: position{line: 22, col: 8, offset: 362}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 22, col: 8, offset: 362}, + exprs: []any{ + &seqExpr{ + pos: position{line: 22, col: 9, offset: 363}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 22, col: 9, offset: 363}, + name: "term", + }, + &choiceExpr{ + pos: position{line: 22, col: 15, offset: 369}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 22, col: 15, offset: 369}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 22, col: 21, offset: 375}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 22, col: 27, offset: 381}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 22, col: 32, offset: 386}, + name: "factor", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 22, col: 40, offset: 394}, + run: (*parser).callonterm10, + }, + }, + }, + &seqExpr{ + pos: position{line: 25, col: 5, offset: 467}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 25, col: 5, offset: 467}, + name: "factor", + }, + &stateCodeExpr{ + pos: position{line: 25, col: 12, offset: 474}, + run: (*parser).callonterm13, + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "factor", + pos: position{line: 30, col: 1, offset: 547}, + expr: &choiceExpr{ + pos: position{line: 30, col: 10, offset: 556}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 30, col: 10, offset: 556}, + exprs: []any{ + &seqExpr{ + pos: position{line: 30, col: 11, offset: 557}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 30, col: 12, offset: 558}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 30, col: 12, offset: 558}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 30, col: 18, offset: 564}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 30, col: 23, offset: 569}, + name: "factor", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 30, col: 31, offset: 577}, + run: (*parser).callonfactor8, + }, + }, + }, + &seqExpr{ + pos: position{line: 33, col: 5, offset: 651}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 33, col: 5, offset: 651}, + name: "atom", + }, + &stateCodeExpr{ + pos: position{line: 33, col: 10, offset: 656}, + run: (*parser).callonfactor11, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "atom", + pos: position{line: 38, col: 1, offset: 729}, + expr: &seqExpr{ + pos: position{line: 38, col: 8, offset: 736}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 38, col: 9, offset: 737}, + expr: &charClassMatcher{ + pos: position{line: 38, col: 9, offset: 737}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &stateCodeExpr{ + pos: position{line: 38, col: 17, offset: 745}, + run: (*parser).callonatom4, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onstart3() error { + + if _, ok := c.state["count"]; !ok { + c.state["count"] = 0 + } + return nil +} + +func (p *parser) callonstart3() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart3() +} + +func (c *current) onstart1() (any, error) { + return c.state["count"], nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1() +} + +func (c *current) onexpr9() error { + c.state["count"] = c.state["count"].(int) + 1 + return nil +} + +func (p *parser) callonexpr9() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr9() +} + +func (c *current) onexpr12() error { + c.state["count"] = c.state["count"].(int) + 3 + return nil +} + +func (p *parser) callonexpr12() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr12() +} + +func (c *current) onterm10() error { + c.state["count"] = c.state["count"].(int) + 7 + return nil +} + +func (p *parser) callonterm10() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm10() +} + +func (c *current) onterm13() error { + c.state["count"] = c.state["count"].(int) + 15 + return nil +} + +func (p *parser) callonterm13() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm13() +} + +func (c *current) onfactor8() error { + c.state["count"] = c.state["count"].(int) + 31 + return nil +} + +func (p *parser) callonfactor8() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor8() +} + +func (c *current) onfactor11() error { + c.state["count"] = c.state["count"].(int) + 63 + return nil +} + +func (p *parser) callonfactor11() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor11() +} + +func (c *current) onatom4() error { + c.state["count"] = c.state["count"].(int) + 127 + return nil +} + +func (p *parser) callonatom4() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onatom4() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + p.restoreState(lastState) + *p.errs = lastErrors + break + } + lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + var ( + val any + ok bool + ) + + if rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + val, ok := p.parseExpr(expr) + + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + return val, ok + } + p.restoreState(state) + } + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion_state/standart/left_recursion_state.go b/test/left_recursion_state/standart/left_recursion_state.go new file mode 100644 index 00000000..cc315ac9 --- /dev/null +++ b/test/left_recursion_state/standart/left_recursion_state.go @@ -0,0 +1,1764 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursionstate + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "start", + pos: position{line: 5, col: 1, offset: 34}, + expr: &actionExpr{ + pos: position{line: 5, col: 9, offset: 42}, + run: (*parser).callonstart1, + expr: &seqExpr{ + pos: position{line: 5, col: 9, offset: 42}, + exprs: []any{ + &stateCodeExpr{ + pos: position{line: 5, col: 9, offset: 42}, + run: (*parser).callonstart3, + }, + &zeroOrOneExpr{ + pos: position{line: 10, col: 3, offset: 128}, + expr: &labeledExpr{ + pos: position{line: 10, col: 4, offset: 129}, + label: "a", + expr: &ruleRefExpr{ + pos: position{line: 10, col: 6, offset: 131}, + name: "expr", + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "expr", + pos: position{line: 14, col: 1, offset: 173}, + expr: &choiceExpr{ + pos: position{line: 14, col: 9, offset: 181}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 14, col: 9, offset: 181}, + exprs: []any{ + &seqExpr{ + pos: position{line: 14, col: 10, offset: 182}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 14, col: 10, offset: 182}, + name: "expr", + }, + &choiceExpr{ + pos: position{line: 14, col: 16, offset: 188}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 14, col: 16, offset: 188}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 14, col: 22, offset: 194}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 14, col: 27, offset: 199}, + name: "term", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 14, col: 33, offset: 205}, + run: (*parser).callonexpr9, + }, + }, + }, + &seqExpr{ + pos: position{line: 17, col: 5, offset: 278}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 17, col: 5, offset: 278}, + name: "term", + }, + &stateCodeExpr{ + pos: position{line: 17, col: 10, offset: 283}, + run: (*parser).callonexpr12, + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "term", + pos: position{line: 22, col: 1, offset: 355}, + expr: &choiceExpr{ + pos: position{line: 22, col: 8, offset: 362}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 22, col: 8, offset: 362}, + exprs: []any{ + &seqExpr{ + pos: position{line: 22, col: 9, offset: 363}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 22, col: 9, offset: 363}, + name: "term", + }, + &choiceExpr{ + pos: position{line: 22, col: 15, offset: 369}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 22, col: 15, offset: 369}, + val: "*", + ignoreCase: false, + want: "\"*\"", + }, + &litMatcher{ + pos: position{line: 22, col: 21, offset: 375}, + val: "/", + ignoreCase: false, + want: "\"/\"", + }, + &litMatcher{ + pos: position{line: 22, col: 27, offset: 381}, + val: "%", + ignoreCase: false, + want: "\"%\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 22, col: 32, offset: 386}, + name: "factor", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 22, col: 40, offset: 394}, + run: (*parser).callonterm10, + }, + }, + }, + &seqExpr{ + pos: position{line: 25, col: 5, offset: 467}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 25, col: 5, offset: 467}, + name: "factor", + }, + &stateCodeExpr{ + pos: position{line: 25, col: 12, offset: 474}, + run: (*parser).callonterm13, + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "factor", + pos: position{line: 30, col: 1, offset: 547}, + expr: &choiceExpr{ + pos: position{line: 30, col: 10, offset: 556}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 30, col: 10, offset: 556}, + exprs: []any{ + &seqExpr{ + pos: position{line: 30, col: 11, offset: 557}, + exprs: []any{ + &choiceExpr{ + pos: position{line: 30, col: 12, offset: 558}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 30, col: 12, offset: 558}, + val: "+", + ignoreCase: false, + want: "\"+\"", + }, + &litMatcher{ + pos: position{line: 30, col: 18, offset: 564}, + val: "-", + ignoreCase: false, + want: "\"-\"", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 30, col: 23, offset: 569}, + name: "factor", + }, + }, + }, + &stateCodeExpr{ + pos: position{line: 30, col: 31, offset: 577}, + run: (*parser).callonfactor8, + }, + }, + }, + &seqExpr{ + pos: position{line: 33, col: 5, offset: 651}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 33, col: 5, offset: 651}, + name: "atom", + }, + &stateCodeExpr{ + pos: position{line: 33, col: 10, offset: 656}, + run: (*parser).callonfactor11, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "atom", + pos: position{line: 38, col: 1, offset: 729}, + expr: &seqExpr{ + pos: position{line: 38, col: 8, offset: 736}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 38, col: 9, offset: 737}, + expr: &charClassMatcher{ + pos: position{line: 38, col: 9, offset: 737}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &stateCodeExpr{ + pos: position{line: 38, col: 17, offset: 745}, + run: (*parser).callonatom4, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onstart3() error { + + if _, ok := c.state["count"]; !ok { + c.state["count"] = 0 + } + return nil +} + +func (p *parser) callonstart3() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart3() +} + +func (c *current) onstart1() (any, error) { + return c.state["count"], nil +} + +func (p *parser) callonstart1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onstart1() +} + +func (c *current) onexpr9() error { + c.state["count"] = c.state["count"].(int) + 1 + return nil +} + +func (p *parser) callonexpr9() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr9() +} + +func (c *current) onexpr12() error { + c.state["count"] = c.state["count"].(int) + 3 + return nil +} + +func (p *parser) callonexpr12() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onexpr12() +} + +func (c *current) onterm10() error { + c.state["count"] = c.state["count"].(int) + 7 + return nil +} + +func (p *parser) callonterm10() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm10() +} + +func (c *current) onterm13() error { + c.state["count"] = c.state["count"].(int) + 15 + return nil +} + +func (p *parser) callonterm13() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onterm13() +} + +func (c *current) onfactor8() error { + c.state["count"] = c.state["count"].(int) + 31 + return nil +} + +func (p *parser) callonfactor8() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor8() +} + +func (c *current) onfactor11() error { + c.state["count"] = c.state["count"].(int) + 63 + return nil +} + +func (p *parser) callonfactor11() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onfactor11() +} + +func (c *current) onatom4() error { + c.state["count"] = c.state["count"].(int) + 127 + return nil +} + +func (p *parser) callonatom4() error { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onatom4() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) + } + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + p.restoreState(lastState) + *p.errs = lastErrors + break + } + lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion_thrownrecover/errors.go b/test/left_recursion_thrownrecover/errors.go new file mode 100644 index 00000000..dd0bee6c --- /dev/null +++ b/test/left_recursion_thrownrecover/errors.go @@ -0,0 +1,31 @@ +package leftrecursionthrownrecover + +// ErrorLister is the public interface to access the inner errors +// included in a errList. +type ErrorLister interface { + Errors() []error +} + +func (e errList) Errors() []error { + return e +} + +// ParserError is the public interface to errors of type parserError. +type ParserError interface { + Error() string + InnerError() error + Pos() (int, int, int) + Expected() []string +} + +func (p *parserError) InnerError() error { + return p.Inner +} + +func (p *parserError) Pos() (line, col, offset int) { + return p.pos.line, p.pos.col, p.pos.offset +} + +func (p *parserError) Expected() []string { + return p.expected +} diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go new file mode 100644 index 00000000..5bb34da5 --- /dev/null +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go @@ -0,0 +1,2587 @@ +// Code generated by pigeon; DO NOT EDIT. + +package leftrecursionthrownrecover + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "Start", + pos: position{line: 6, col: 1, offset: 41}, + expr: &andCodeExpr{ + pos: position{line: 6, col: 9, offset: 49}, + run: (*parser).callonStart1, + }, + leader: false, + leftRecursive: false, + }, + { + name: "case01", + pos: position{line: 11, col: 1, offset: 108}, + expr: &actionExpr{ + pos: position{line: 11, col: 10, offset: 117}, + run: (*parser).calloncase011, + expr: &labeledExpr{ + pos: position{line: 11, col: 10, offset: 117}, + label: "case01", + expr: &ruleRefExpr{ + pos: position{line: 11, col: 17, offset: 124}, + name: "MultiLabelRecover", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "MultiLabelRecover", + pos: position{line: 13, col: 1, offset: 167}, + expr: &recoveryExpr{ + pos: position{line: 13, col: 21, offset: 187}, + expr: &ruleRefExpr{ + pos: position{line: 13, col: 21, offset: 187}, + name: "number", + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 13, col: 51, offset: 217}, + name: "ErrNonNumber", + }, + failureLabel: []string{ + "errAlpha", + "errOther", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "number", + pos: position{line: 15, col: 1, offset: 231}, + expr: &choiceExpr{ + pos: position{line: 15, col: 10, offset: 240}, + alternatives: []any{ + ¬Expr{ + pos: position{line: 15, col: 10, offset: 240}, + expr: &anyMatcher{ + line: 15, col: 11, offset: 241, + }, + }, + &actionExpr{ + pos: position{line: 15, col: 15, offset: 245}, + run: (*parser).callonnumber4, + expr: &seqExpr{ + pos: position{line: 15, col: 16, offset: 246}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 15, col: 16, offset: 246}, + label: "n", + expr: &ruleRefExpr{ + pos: position{line: 15, col: 18, offset: 248}, + name: "number", + }, + }, + &labeledExpr{ + pos: position{line: 15, col: 25, offset: 255}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 15, col: 27, offset: 257}, + name: "digit", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 17, col: 5, offset: 310}, + run: (*parser).callonnumber10, + expr: &labeledExpr{ + pos: position{line: 17, col: 5, offset: 310}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 17, col: 7, offset: 312}, + name: "digit", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "digit", + pos: position{line: 21, col: 1, offset: 350}, + expr: &choiceExpr{ + pos: position{line: 21, col: 9, offset: 358}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 21, col: 9, offset: 358}, + run: (*parser).callondigit2, + expr: &charClassMatcher{ + pos: position{line: 21, col: 9, offset: 358}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &actionExpr{ + pos: position{line: 23, col: 5, offset: 401}, + run: (*parser).callondigit4, + expr: &labeledExpr{ + pos: position{line: 23, col: 5, offset: 401}, + label: "x", + expr: &seqExpr{ + pos: position{line: 23, col: 9, offset: 405}, + exprs: []any{ + &andExpr{ + pos: position{line: 23, col: 9, offset: 405}, + expr: &charClassMatcher{ + pos: position{line: 23, col: 10, offset: 406}, + val: "[a-z]", + ranges: []rune{'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + &throwExpr{ + pos: position{line: 23, col: 16, offset: 412}, + label: "errAlpha", + }, + }, + }, + }, + }, + &throwExpr{ + pos: position{line: 25, col: 5, offset: 461}, + label: "errOther", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrNonNumber", + pos: position{line: 27, col: 1, offset: 474}, + expr: &actionExpr{ + pos: position{line: 27, col: 16, offset: 489}, + run: (*parser).callonErrNonNumber1, + expr: &seqExpr{ + pos: position{line: 27, col: 16, offset: 489}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 27, col: 16, offset: 489}, + run: (*parser).callonErrNonNumber3, + }, + &zeroOrMoreExpr{ + pos: position{line: 29, col: 3, offset: 544}, + expr: &seqExpr{ + pos: position{line: 29, col: 5, offset: 546}, + exprs: []any{ + ¬Expr{ + pos: position{line: 29, col: 5, offset: 546}, + expr: &charClassMatcher{ + pos: position{line: 29, col: 6, offset: 547}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 29, col: 12, offset: 553, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "case02", + pos: position{line: 34, col: 1, offset: 615}, + expr: &choiceExpr{ + pos: position{line: 34, col: 11, offset: 625}, + alternatives: []any{ + &ruleRefExpr{ + pos: position{line: 34, col: 11, offset: 625}, + name: "ThrowUndefLabel", + }, + &andCodeExpr{ + pos: position{line: 34, col: 29, offset: 643}, + run: (*parser).calloncase023, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ThrowUndefLabel", + pos: position{line: 36, col: 1, offset: 702}, + expr: &seqExpr{ + pos: position{line: 36, col: 19, offset: 720}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 36, col: 19, offset: 720}, + name: "ThrowUndefLabel", + }, + &throwExpr{ + pos: position{line: 36, col: 35, offset: 736}, + label: "undeflabel", + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "case03", + pos: position{line: 41, col: 1, offset: 780}, + expr: &actionExpr{ + pos: position{line: 41, col: 10, offset: 789}, + run: (*parser).calloncase031, + expr: &labeledExpr{ + pos: position{line: 41, col: 10, offset: 789}, + label: "case03", + expr: &ruleRefExpr{ + pos: position{line: 41, col: 17, offset: 796}, + name: "OuterRecover03", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "OuterRecover03", + pos: position{line: 43, col: 1, offset: 835}, + expr: &recoveryExpr{ + pos: position{line: 43, col: 18, offset: 852}, + expr: &recoveryExpr{ + pos: position{line: 43, col: 18, offset: 852}, + expr: &ruleRefExpr{ + pos: position{line: 43, col: 18, offset: 852}, + name: "InnerRecover03", + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 43, col: 66, offset: 900}, + name: "ErrAlphaOuter03", + }, + failureLabel: []string{ + "errAlphaLower", + "errAlphaUpper", + }, + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 43, col: 95, offset: 929}, + name: "ErrOtherOuter03", + }, + failureLabel: []string{ + "errOther", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "InnerRecover03", + pos: position{line: 45, col: 1, offset: 946}, + expr: &recoveryExpr{ + pos: position{line: 45, col: 18, offset: 963}, + expr: &ruleRefExpr{ + pos: position{line: 45, col: 18, offset: 963}, + name: "number03", + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 45, col: 45, offset: 990}, + name: "ErrAlphaInner03", + }, + failureLabel: []string{ + "errAlphaLower", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "number03", + pos: position{line: 47, col: 1, offset: 1007}, + expr: &choiceExpr{ + pos: position{line: 47, col: 12, offset: 1018}, + alternatives: []any{ + ¬Expr{ + pos: position{line: 47, col: 12, offset: 1018}, + expr: &anyMatcher{ + line: 47, col: 13, offset: 1019, + }, + }, + &actionExpr{ + pos: position{line: 47, col: 17, offset: 1023}, + run: (*parser).callonnumber034, + expr: &seqExpr{ + pos: position{line: 47, col: 18, offset: 1024}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 47, col: 18, offset: 1024}, + label: "n", + expr: &ruleRefExpr{ + pos: position{line: 47, col: 20, offset: 1026}, + name: "number03", + }, + }, + &labeledExpr{ + pos: position{line: 47, col: 29, offset: 1035}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 47, col: 31, offset: 1037}, + name: "digit03", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 49, col: 5, offset: 1092}, + run: (*parser).callonnumber0310, + expr: &labeledExpr{ + pos: position{line: 49, col: 5, offset: 1092}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 49, col: 7, offset: 1094}, + name: "digit03", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "digit03", + pos: position{line: 53, col: 1, offset: 1134}, + expr: &choiceExpr{ + pos: position{line: 53, col: 11, offset: 1144}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 53, col: 11, offset: 1144}, + run: (*parser).callondigit032, + expr: &charClassMatcher{ + pos: position{line: 53, col: 11, offset: 1144}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &actionExpr{ + pos: position{line: 55, col: 5, offset: 1187}, + run: (*parser).callondigit034, + expr: &labeledExpr{ + pos: position{line: 55, col: 5, offset: 1187}, + label: "x", + expr: &seqExpr{ + pos: position{line: 55, col: 9, offset: 1191}, + exprs: []any{ + &andExpr{ + pos: position{line: 55, col: 9, offset: 1191}, + expr: &charClassMatcher{ + pos: position{line: 55, col: 10, offset: 1192}, + val: "[a-z]", + ranges: []rune{'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + &throwExpr{ + pos: position{line: 55, col: 16, offset: 1198}, + label: "errAlphaLower", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 57, col: 5, offset: 1252}, + run: (*parser).callondigit0310, + expr: &labeledExpr{ + pos: position{line: 57, col: 5, offset: 1252}, + label: "x", + expr: &seqExpr{ + pos: position{line: 57, col: 9, offset: 1256}, + exprs: []any{ + &andExpr{ + pos: position{line: 57, col: 9, offset: 1256}, + expr: &charClassMatcher{ + pos: position{line: 57, col: 10, offset: 1257}, + val: "[A-Z]", + ranges: []rune{'A', 'Z'}, + ignoreCase: false, + inverted: false, + }, + }, + &throwExpr{ + pos: position{line: 57, col: 16, offset: 1263}, + label: "errAlphaUpper", + }, + }, + }, + }, + }, + &throwExpr{ + pos: position{line: 59, col: 5, offset: 1317}, + label: "errOther", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrAlphaInner03", + pos: position{line: 61, col: 1, offset: 1330}, + expr: &actionExpr{ + pos: position{line: 61, col: 19, offset: 1348}, + run: (*parser).callonErrAlphaInner031, + expr: &seqExpr{ + pos: position{line: 61, col: 19, offset: 1348}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 61, col: 19, offset: 1348}, + run: (*parser).callonErrAlphaInner033, + }, + &zeroOrMoreExpr{ + pos: position{line: 63, col: 3, offset: 1424}, + expr: &seqExpr{ + pos: position{line: 63, col: 5, offset: 1426}, + exprs: []any{ + ¬Expr{ + pos: position{line: 63, col: 5, offset: 1426}, + expr: &charClassMatcher{ + pos: position{line: 63, col: 6, offset: 1427}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 63, col: 12, offset: 1433, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrAlphaOuter03", + pos: position{line: 65, col: 1, offset: 1459}, + expr: &actionExpr{ + pos: position{line: 65, col: 19, offset: 1477}, + run: (*parser).callonErrAlphaOuter031, + expr: &seqExpr{ + pos: position{line: 65, col: 19, offset: 1477}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 65, col: 19, offset: 1477}, + run: (*parser).callonErrAlphaOuter033, + }, + &zeroOrMoreExpr{ + pos: position{line: 67, col: 3, offset: 1553}, + expr: &seqExpr{ + pos: position{line: 67, col: 5, offset: 1555}, + exprs: []any{ + ¬Expr{ + pos: position{line: 67, col: 5, offset: 1555}, + expr: &charClassMatcher{ + pos: position{line: 67, col: 6, offset: 1556}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 67, col: 12, offset: 1562, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrOtherOuter03", + pos: position{line: 69, col: 1, offset: 1588}, + expr: &actionExpr{ + pos: position{line: 69, col: 19, offset: 1606}, + run: (*parser).callonErrOtherOuter031, + expr: &seqExpr{ + pos: position{line: 69, col: 19, offset: 1606}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 69, col: 19, offset: 1606}, + run: (*parser).callonErrOtherOuter033, + }, + &zeroOrMoreExpr{ + pos: position{line: 71, col: 3, offset: 1677}, + expr: &seqExpr{ + pos: position{line: 71, col: 5, offset: 1679}, + exprs: []any{ + ¬Expr{ + pos: position{line: 71, col: 5, offset: 1679}, + expr: &charClassMatcher{ + pos: position{line: 71, col: 6, offset: 1680}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 71, col: 12, offset: 1686, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "case04", + pos: position{line: 76, col: 1, offset: 1771}, + expr: &actionExpr{ + pos: position{line: 76, col: 10, offset: 1780}, + run: (*parser).calloncase041, + expr: &labeledExpr{ + pos: position{line: 76, col: 10, offset: 1780}, + label: "case04", + expr: &ruleRefExpr{ + pos: position{line: 76, col: 17, offset: 1787}, + name: "OuterRecover04", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "OuterRecover04", + pos: position{line: 78, col: 1, offset: 1826}, + expr: &recoveryExpr{ + pos: position{line: 78, col: 18, offset: 1843}, + expr: &recoveryExpr{ + pos: position{line: 78, col: 18, offset: 1843}, + expr: &ruleRefExpr{ + pos: position{line: 78, col: 18, offset: 1843}, + name: "InnerRecover04", + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 78, col: 66, offset: 1891}, + name: "ErrAlphaOuter04", + }, + failureLabel: []string{ + "errAlphaLower", + "errAlphaUpper", + }, + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 78, col: 95, offset: 1920}, + name: "ErrOtherOuter04", + }, + failureLabel: []string{ + "errOther", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "InnerRecover04", + pos: position{line: 80, col: 1, offset: 1937}, + expr: &recoveryExpr{ + pos: position{line: 80, col: 18, offset: 1954}, + expr: &ruleRefExpr{ + pos: position{line: 80, col: 18, offset: 1954}, + name: "number04", + }, + recoverExpr: &ruleRefExpr{ + pos: position{line: 80, col: 45, offset: 1981}, + name: "ErrAlphaInner04", + }, + failureLabel: []string{ + "errAlphaLower", + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "number04", + pos: position{line: 82, col: 1, offset: 1998}, + expr: &choiceExpr{ + pos: position{line: 82, col: 12, offset: 2009}, + alternatives: []any{ + ¬Expr{ + pos: position{line: 82, col: 12, offset: 2009}, + expr: &anyMatcher{ + line: 82, col: 13, offset: 2010, + }, + }, + &actionExpr{ + pos: position{line: 82, col: 17, offset: 2014}, + run: (*parser).callonnumber044, + expr: &seqExpr{ + pos: position{line: 82, col: 18, offset: 2015}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 82, col: 18, offset: 2015}, + label: "n", + expr: &ruleRefExpr{ + pos: position{line: 82, col: 20, offset: 2017}, + name: "number04", + }, + }, + &labeledExpr{ + pos: position{line: 82, col: 29, offset: 2026}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 82, col: 31, offset: 2028}, + name: "digit04", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 84, col: 5, offset: 2083}, + run: (*parser).callonnumber0410, + expr: &labeledExpr{ + pos: position{line: 84, col: 5, offset: 2083}, + label: "d", + expr: &ruleRefExpr{ + pos: position{line: 84, col: 7, offset: 2085}, + name: "digit04", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "digit04", + pos: position{line: 88, col: 1, offset: 2125}, + expr: &choiceExpr{ + pos: position{line: 88, col: 11, offset: 2135}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 88, col: 11, offset: 2135}, + run: (*parser).callondigit042, + expr: &charClassMatcher{ + pos: position{line: 88, col: 11, offset: 2135}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &actionExpr{ + pos: position{line: 90, col: 5, offset: 2178}, + run: (*parser).callondigit044, + expr: &labeledExpr{ + pos: position{line: 90, col: 5, offset: 2178}, + label: "x", + expr: &seqExpr{ + pos: position{line: 90, col: 9, offset: 2182}, + exprs: []any{ + &andExpr{ + pos: position{line: 90, col: 9, offset: 2182}, + expr: &charClassMatcher{ + pos: position{line: 90, col: 10, offset: 2183}, + val: "[a-z]", + ranges: []rune{'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + &throwExpr{ + pos: position{line: 90, col: 16, offset: 2189}, + label: "errAlphaLower", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 92, col: 5, offset: 2243}, + run: (*parser).callondigit0410, + expr: &labeledExpr{ + pos: position{line: 92, col: 5, offset: 2243}, + label: "x", + expr: &seqExpr{ + pos: position{line: 92, col: 9, offset: 2247}, + exprs: []any{ + &andExpr{ + pos: position{line: 92, col: 9, offset: 2247}, + expr: &charClassMatcher{ + pos: position{line: 92, col: 10, offset: 2248}, + val: "[A-Z]", + ranges: []rune{'A', 'Z'}, + ignoreCase: false, + inverted: false, + }, + }, + &throwExpr{ + pos: position{line: 92, col: 16, offset: 2254}, + label: "errAlphaUpper", + }, + }, + }, + }, + }, + &throwExpr{ + pos: position{line: 94, col: 5, offset: 2308}, + label: "errOther", + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrAlphaInner04", + pos: position{line: 96, col: 1, offset: 2321}, + expr: &andCodeExpr{ + pos: position{line: 96, col: 19, offset: 2339}, + run: (*parser).callonErrAlphaInner041, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrAlphaOuter04", + pos: position{line: 100, col: 1, offset: 2367}, + expr: &actionExpr{ + pos: position{line: 100, col: 19, offset: 2385}, + run: (*parser).callonErrAlphaOuter041, + expr: &seqExpr{ + pos: position{line: 100, col: 19, offset: 2385}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 100, col: 19, offset: 2385}, + run: (*parser).callonErrAlphaOuter043, + }, + &zeroOrMoreExpr{ + pos: position{line: 102, col: 3, offset: 2452}, + expr: &seqExpr{ + pos: position{line: 102, col: 5, offset: 2454}, + exprs: []any{ + ¬Expr{ + pos: position{line: 102, col: 5, offset: 2454}, + expr: &charClassMatcher{ + pos: position{line: 102, col: 6, offset: 2455}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 102, col: 12, offset: 2461, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "ErrOtherOuter04", + pos: position{line: 104, col: 1, offset: 2487}, + expr: &actionExpr{ + pos: position{line: 104, col: 19, offset: 2505}, + run: (*parser).callonErrOtherOuter041, + expr: &seqExpr{ + pos: position{line: 104, col: 19, offset: 2505}, + exprs: []any{ + &andCodeExpr{ + pos: position{line: 104, col: 19, offset: 2505}, + run: (*parser).callonErrOtherOuter043, + }, + &zeroOrMoreExpr{ + pos: position{line: 106, col: 3, offset: 2576}, + expr: &seqExpr{ + pos: position{line: 106, col: 5, offset: 2578}, + exprs: []any{ + ¬Expr{ + pos: position{line: 106, col: 5, offset: 2578}, + expr: &charClassMatcher{ + pos: position{line: 106, col: 6, offset: 2579}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &anyMatcher{ + line: 106, col: 12, offset: 2585, + }, + }, + }, + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onStart1() (bool, error) { + return false, nil +} + +func (p *parser) callonStart1() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onStart1() +} + +func (c *current) oncase011(case01 any) (any, error) { + return case01, nil +} + +func (p *parser) calloncase011() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.oncase011(stack["case01"]) +} + +func (c *current) onnumber4(n, d any) (any, error) { + return n.(string) + d.(string), nil +} + +func (p *parser) callonnumber4() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber4(stack["n"], stack["d"]) +} + +func (c *current) onnumber10(d any) (any, error) { + return d.(string), nil +} + +func (p *parser) callonnumber10() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber10(stack["d"]) +} + +func (c *current) ondigit2() (any, error) { + return string(c.text), nil +} + +func (p *parser) callondigit2() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit2() +} + +func (c *current) ondigit4(x any) (any, error) { + return x.([]any)[1], nil +} + +func (p *parser) callondigit4() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit4(stack["x"]) +} + +func (c *current) onErrNonNumber3() (bool, error) { + return true, errors.New("expecting a number") +} + +func (p *parser) callonErrNonNumber3() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrNonNumber3() +} + +func (c *current) onErrNonNumber1() (any, error) { + return "?", nil +} + +func (p *parser) callonErrNonNumber1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrNonNumber1() +} + +func (c *current) oncase023() (bool, error) { + return false, errors.New("Throwed undefined label") +} + +func (p *parser) calloncase023() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.oncase023() +} + +func (c *current) oncase031(case03 any) (any, error) { + return case03, nil +} + +func (p *parser) calloncase031() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.oncase031(stack["case03"]) +} + +func (c *current) onnumber034(n, d any) (any, error) { + return n.(string) + d.(string), nil +} + +func (p *parser) callonnumber034() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber034(stack["n"], stack["d"]) +} + +func (c *current) onnumber0310(d any) (any, error) { + return d.(string), nil +} + +func (p *parser) callonnumber0310() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber0310(stack["d"]) +} + +func (c *current) ondigit032() (any, error) { + return string(c.text), nil +} + +func (p *parser) callondigit032() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit032() +} + +func (c *current) ondigit034(x any) (any, error) { + return x.([]any)[1], nil +} + +func (p *parser) callondigit034() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit034(stack["x"]) +} + +func (c *current) ondigit0310(x any) (any, error) { + return x.([]any)[1], nil +} + +func (p *parser) callondigit0310() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit0310(stack["x"]) +} + +func (c *current) onErrAlphaInner033() (bool, error) { + return true, errors.New("expecting a number, got lower case char") +} + +func (p *parser) callonErrAlphaInner033() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaInner033() +} + +func (c *current) onErrAlphaInner031() (any, error) { + return "<", nil +} + +func (p *parser) callonErrAlphaInner031() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaInner031() +} + +func (c *current) onErrAlphaOuter033() (bool, error) { + return true, errors.New("expecting a number, got upper case char") +} + +func (p *parser) callonErrAlphaOuter033() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaOuter033() +} + +func (c *current) onErrAlphaOuter031() (any, error) { + return ">", nil +} + +func (p *parser) callonErrAlphaOuter031() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaOuter031() +} + +func (c *current) onErrOtherOuter033() (bool, error) { + return true, errors.New("expecting a number, got a non-char") +} + +func (p *parser) callonErrOtherOuter033() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrOtherOuter033() +} + +func (c *current) onErrOtherOuter031() (any, error) { + return "?", nil +} + +func (p *parser) callonErrOtherOuter031() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrOtherOuter031() +} + +func (c *current) oncase041(case04 any) (any, error) { + return case04, nil +} + +func (p *parser) calloncase041() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.oncase041(stack["case04"]) +} + +func (c *current) onnumber044(n, d any) (any, error) { + return n.(string) + d.(string), nil +} + +func (p *parser) callonnumber044() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber044(stack["n"], stack["d"]) +} + +func (c *current) onnumber0410(d any) (any, error) { + return d.(string), nil +} + +func (p *parser) callonnumber0410() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onnumber0410(stack["d"]) +} + +func (c *current) ondigit042() (any, error) { + return string(c.text), nil +} + +func (p *parser) callondigit042() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit042() +} + +func (c *current) ondigit044(x any) (any, error) { + return x.([]any)[1], nil +} + +func (p *parser) callondigit044() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit044(stack["x"]) +} + +func (c *current) ondigit0410(x any) (any, error) { + return x.([]any)[1], nil +} + +func (p *parser) callondigit0410() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.ondigit0410(stack["x"]) +} + +func (c *current) onErrAlphaInner041() (bool, error) { + return false, nil +} + +func (p *parser) callonErrAlphaInner041() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaInner041() +} + +func (c *current) onErrAlphaOuter043() (bool, error) { + return true, errors.New("expecting a number, got a char") +} + +func (p *parser) callonErrAlphaOuter043() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaOuter043() +} + +func (c *current) onErrAlphaOuter041() (any, error) { + return "x", nil +} + +func (p *parser) callonErrAlphaOuter041() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrAlphaOuter041() +} + +func (c *current) onErrOtherOuter043() (bool, error) { + return true, errors.New("expecting a number, got a non-char") +} + +func (p *parser) callonErrOtherOuter043() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrOtherOuter043() +} + +func (c *current) onErrOtherOuter041() (any, error) { + return "?", nil +} + +func (p *parser) callonErrOtherOuter041() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onErrOtherOuter041() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { // nolint: deadcode + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { // nolint: deadcode + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +// nolint: structcheck +type grammar struct { + pos position + rules []*rule +} + +// nolint: structcheck +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +// nolint: structcheck +type choiceExpr struct { + pos position + alternatives []any +} + +// nolint: structcheck +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +// nolint: structcheck +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +// nolint: structcheck +type seqExpr struct { + pos position + exprs []any +} + +// nolint: structcheck +type throwExpr struct { + pos position + label string +} + +// nolint: structcheck +type labeledExpr struct { + pos position + label string + expr any +} + +// nolint: structcheck +type expr struct { + pos position + expr any +} + +type ( + andExpr expr // nolint: structcheck + notExpr expr // nolint: structcheck + zeroOrOneExpr expr // nolint: structcheck + zeroOrMoreExpr expr // nolint: structcheck + oneOrMoreExpr expr // nolint: structcheck +) + +// nolint: structcheck +type ruleRefExpr struct { + pos position + name string +} + +// nolint: structcheck +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +// nolint: structcheck +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +// nolint: structcheck +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +// nolint: structcheck +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position // nolint: structcheck + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +// nolint: structcheck,deadcode +type resultTuple struct { + v any + b bool + end savepoint +} + +// nolint: varcheck +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +// nolint: structcheck,maligned +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +// nolint: gocyclo +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) + } + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + p.restoreState(lastState) + *p.errs = lastErrors + break + } + lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +// nolint: gocyclo +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +// nolint: gocyclo +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover.peg b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.peg new file mode 100644 index 00000000..bdd5f72f --- /dev/null +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.peg @@ -0,0 +1,106 @@ +{ +package leftrecursionthrownrecover + +} + +Start = &{return false, nil} + + +// Case 01: Multiple Label Recover + +case01 = case01:MultiLabelRecover { return case01, nil } + +MultiLabelRecover = number //{errAlpha, errOther} ErrNonNumber + +number = !. / (n:number d:digit) { + return n.(string) + d.(string), nil +} / d:digit { + return d.(string), nil +} + +digit = [0-9] { + return string(c.text), nil +} / x:( &[a-z] %{errAlpha} ) { + return x.([]any)[1], nil +} / %{errOther} + +ErrNonNumber = &{ + return true, errors.New("expecting a number") +} ( ![0-9] . )* { return "?", nil } + + +// Case 02: Throw Undefined Label + +case02 = (ThrowUndefLabel / &{ return false, errors.New("Throwed undefined label") }) + +ThrowUndefLabel = ThrowUndefLabel %{undeflabel} + + +// Case 03: Nested Recover + +case03 = case03:OuterRecover03 { return case03, nil } + +OuterRecover03 = InnerRecover03 //{errAlphaLower, errAlphaUpper} ErrAlphaOuter03 //{errOther} ErrOtherOuter03 + +InnerRecover03 = number03 //{errAlphaLower} ErrAlphaInner03 + +number03 = !. / (n:number03 d:digit03) { + return n.(string) + d.(string), nil +} / d:digit03 { + return d.(string), nil +} + +digit03 = [0-9] { + return string(c.text), nil +} / x:( &[a-z] %{errAlphaLower} ) { + return x.([]any)[1], nil +} / x:( &[A-Z] %{errAlphaUpper} ) { + return x.([]any)[1], nil +} / %{errOther} + +ErrAlphaInner03 = &{ + return true, errors.New("expecting a number, got lower case char") +} ( ![0-9] . )* { return "<", nil } + +ErrAlphaOuter03 = &{ + return true, errors.New("expecting a number, got upper case char") +} ( ![0-9] . )* { return ">", nil } + +ErrOtherOuter03 = &{ + return true, errors.New("expecting a number, got a non-char") +} ( ![0-9] . )* { return "?", nil } + + +// Case 04: Nested Recover, which fails in inner recover + +case04 = case04:OuterRecover04 { return case04, nil } + +OuterRecover04 = InnerRecover04 //{errAlphaLower, errAlphaUpper} ErrAlphaOuter04 //{errOther} ErrOtherOuter04 + +InnerRecover04 = number04 //{errAlphaLower} ErrAlphaInner04 + +number04 = !. / (n:number04 d:digit04) { + return n.(string) + d.(string), nil +} / d:digit04 { + return d.(string), nil +} + +digit04 = [0-9] { + return string(c.text), nil +} / x:( &[a-z] %{errAlphaLower} ) { + return x.([]any)[1], nil +} / x:( &[A-Z] %{errAlphaUpper} ) { + return x.([]any)[1], nil +} / %{errOther} + +ErrAlphaInner04 = &{ + return false, nil +} + +ErrAlphaOuter04 = &{ + return true, errors.New("expecting a number, got a char") +} ( ![0-9] . )* { return "x", nil } + +ErrOtherOuter04 = &{ + return true, errors.New("expecting a number, got a non-char") +} ( ![0-9] . )* { return "?", nil } diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go new file mode 100644 index 00000000..9e12223c --- /dev/null +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go @@ -0,0 +1,204 @@ +package leftrecursionthrownrecover_test + +import ( + "reflect" + "testing" + + leftrecursionthrownrecover "github.com/mna/pigeon/test/left_recursion_thrownrecover" +) + +func TestLeftRecursionWithThrowAndRecover(t *testing.T) { + t.Parallel() + + type want struct { + captures any + errors []string + } + + cases := []struct { + name string + entrypoint string + input string + want want + }{ + // Case 01: Recover multiple labels + { + name: "Case 01: Recover multiple labels[correct]", + entrypoint: "case01", + input: "123", + want: want{captures: "123"}, + }, + { + name: "Case 01: Recover multiple labels[second character is not a number]", + entrypoint: "case01", + input: "1a3", + want: want{ + captures: "1?3", + errors: []string{ + "1:2 (1): rule ErrNonNumber: expecting a number", + }, + }, + }, + { + name: "Case 01: Recover multiple labels[third character is not a number]", + entrypoint: "case01", + input: "11+3", + want: want{ + captures: "11?3", + errors: []string{ + "1:3 (2): rule ErrNonNumber: expecting a number", + }, + }, + }, + + // Case 02: Throw a undefined label + { + name: "Case 02: Throw a undefined label", + entrypoint: "case02", + input: "", + want: want{ + captures: nil, + errors: []string{ + "1:1 (0): rule case02: Throwed undefined label", + }, + }, + }, + + // Case 03: Nested Recover + { + name: "Case 03: Nested Recover[correct]", + entrypoint: "case03", + input: "123", + want: want{captures: "123"}, + }, + { + name: "Case 03: Nested Recover[second character is lower case char]", + entrypoint: "case03", + input: "1a3", + want: want{ + captures: "1<3", + errors: []string{ + "1:2 (1): rule ErrAlphaInner03: expecting a number, got lower case char", + }, + }, + }, + { + name: "Case 03: Nested Recover[third character is upper case char]", + entrypoint: "case03", + input: "11A3", + want: want{ + captures: "11>3", + errors: []string{ + "1:3 (2): rule ErrAlphaOuter03: expecting a number, got upper case char", + }, + }, + }, + { + name: "Case 03: Nested Recover[fourth character is non-char]", + entrypoint: "case03", + input: "111+3", + want: want{ + captures: "111?3", + errors: []string{ + "1:4 (3): rule ErrOtherOuter03: expecting a number, got a non-char", + }, + }, + }, + + // Case 04: Nested Recover, which fails in inner recover + { + name: "Case 04: Nested Recover, which fails in inner recover[correct]", + entrypoint: "case04", + input: "123", + want: want{captures: "123"}, + }, + { + name: "Case 04: Nested Recover, which fails in inner recover[second character is lower case char]", + entrypoint: "case04", + input: "1a3", + want: want{ + captures: "1x3", + errors: []string{ + "1:2 (1): rule ErrAlphaOuter04: expecting a number, got a char", + }, + }, + }, + { + name: "Case 04: Nested Recover, which fails in inner recover[third character is upper case char]", + entrypoint: "case04", + input: "11A3", + want: want{ + captures: "11x3", + errors: []string{ + "1:3 (2): rule ErrAlphaOuter04: expecting a number, got a char", + }, + }, + }, + { + name: "Case 04: Nested Recover, which fails in inner recover[fourth character is non-char]", + entrypoint: "case04", + input: "111+3", + want: want{ + captures: "111?3", + errors: []string{ + "1:4 (3): rule ErrOtherOuter04: expecting a number, got a non-char", + }, + }, + }, + } + for _, testCase := range cases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + got, err := leftrecursionthrownrecover.Parse( + "", []byte(testCase.input), + leftrecursionthrownrecover.Entrypoint(testCase.entrypoint)) + if testCase.want.errors == nil && err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.input, err) + } + if testCase.want.errors != nil && err == nil { + t.Fatalf( + "for input %q got no error, but expect to parse with errors: %s", + testCase.input, testCase.want.errors) + } + if !reflect.DeepEqual(got, testCase.want.captures) { + t.Errorf( + "for input %q want %s, got %s", testCase.input, testCase.want.captures, got) + } + if err != nil { + errorLister, ok := err.(leftrecursionthrownrecover.ErrorLister) + if !ok { + t.FailNow() + } + list := errorLister.Errors() + if len(list) != len(testCase.want.errors) { + t.Errorf( + "for input %q want %d error(s), got %d", + testCase.input, len(testCase.want.errors), len(list)) + t.Logf("expected errors:\n") + for _, ee := range testCase.want.errors { + t.Logf("- %s\n", ee) + } + t.Logf("got errors:\n") + for _, ee := range list { + t.Logf("- %s\n", ee) + } + t.FailNow() + } + for index, err := range list { + pe, ok := err.(leftrecursionthrownrecover.ParserError) + if !ok { + t.FailNow() + } + if pe.Error() != testCase.want.errors[index] { + t.Errorf( + "for input %q want %dth error to be %s, got %s", + testCase.input, index+1, testCase.want.errors[index], pe) + } + } + } + }) + } +} From 31bf44861743c8a44653e0227d5acaae8530ab4e Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Mon, 10 Jul 2023 10:12:24 +0300 Subject: [PATCH 11/14] Use errors.As --- .../left_recursion_labeled_failures_test.go | 14 ++++++++------ .../left_recursion_thrownrecover_test.go | 17 ++++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go index b5938994..7889a05f 100644 --- a/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go @@ -1,6 +1,7 @@ package leftrecursionlabeledfailures_test import ( + "errors" "reflect" "testing" @@ -118,8 +119,8 @@ func TestLeftRecursionWithLabeledFailures(t *testing.T) { testCase.input, testCase.want.captures, got) } if err != nil { - errorLister, ok := err.(leftrecursionlabeledfailures.ErrorLister) - if !ok { + var errorLister leftrecursionlabeledfailures.ErrorLister + if !errors.As(err, &errorLister) { t.FailNow() } list := errorLister.Errors() @@ -138,14 +139,15 @@ func TestLeftRecursionWithLabeledFailures(t *testing.T) { t.FailNow() } for index, err := range list { - pe, ok := err.(leftrecursionlabeledfailures.ParserError) - if !ok { + var parserError leftrecursionlabeledfailures.ParserError + if !errors.As(err, &parserError) { t.FailNow() } - if pe.Error() != testCase.want.errors[index] { + if parserError.Error() != testCase.want.errors[index] { t.Errorf( "for input %q want %dth error to be %s, got %s", - testCase.input, index+1, testCase.want.errors[index], pe) + testCase.input, index+1, + testCase.want.errors[index], parserError) } } } diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go index 9e12223c..1cbe0bc8 100644 --- a/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go @@ -1,6 +1,7 @@ package leftrecursionthrownrecover_test import ( + "errors" "reflect" "testing" @@ -165,11 +166,12 @@ func TestLeftRecursionWithThrowAndRecover(t *testing.T) { } if !reflect.DeepEqual(got, testCase.want.captures) { t.Errorf( - "for input %q want %s, got %s", testCase.input, testCase.want.captures, got) + "for input %q want %s, got %s", + testCase.input, testCase.want.captures, got) } if err != nil { - errorLister, ok := err.(leftrecursionthrownrecover.ErrorLister) - if !ok { + var errorLister leftrecursionthrownrecover.ErrorLister + if !errors.As(err, &errorLister) { t.FailNow() } list := errorLister.Errors() @@ -188,14 +190,15 @@ func TestLeftRecursionWithThrowAndRecover(t *testing.T) { t.FailNow() } for index, err := range list { - pe, ok := err.(leftrecursionthrownrecover.ParserError) - if !ok { + var parserError leftrecursionthrownrecover.ParserError + if !errors.As(err, &parserError) { t.FailNow() } - if pe.Error() != testCase.want.errors[index] { + if parserError.Error() != testCase.want.errors[index] { t.Errorf( "for input %q want %dth error to be %s, got %s", - testCase.input, index+1, testCase.want.errors[index], pe) + testCase.input, index+1, + testCase.want.errors[index], parserError) } } } From fabdceb73ef5895384152a682e4f0b75309e8d85 Mon Sep 17 00:00:00 2001 From: "k.molodyakov" Date: Fri, 21 Jul 2023 19:44:01 +0300 Subject: [PATCH 12/14] Fix Memoize mode --- builder/generated_static_code.go | 9 ++ builder/static_code.go | 9 ++ test/issue_70b/issue_70b.go | 5 +- test/left_recursion/left_recursion_test.go | 133 +++++++++++++----- .../standart/leftrecursion/left_recursion.go | 5 +- .../left_recursion_labeled_failures.go | 5 +- .../left_recursion_labeled_failures_test.go | 89 ++++++------ .../left_recursion_state_test.go | 46 +++--- .../standart/left_recursion_state.go | 5 +- .../left_recursion_thrownrecover.go | 5 +- .../left_recursion_thrownrecover_test.go | 106 ++++++++------ 11 files changed, 271 insertions(+), 146 deletions(-) diff --git a/builder/generated_static_code.go b/builder/generated_static_code.go index ca61a9fd..ede68317 100644 --- a/builder/generated_static_code.go +++ b/builder/generated_static_code.go @@ -1034,7 +1034,12 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { // ==template== {{ if not .Optimize }} var pt savepoint + // ==template== {{ if .LeftRecursion }} + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { + // {{ else }} if p.memoize { + // {{ end }} ==template== res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1047,7 +1052,11 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) // ==template== {{ if not .Optimize }} + // ==template== {{ if .LeftRecursion }} + if p.memoize && !isLeftRecusion { + // {{ else }} if p.memoize { + // {{ end }} ==template== p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } // {{ end }} ==template== diff --git a/builder/static_code.go b/builder/static_code.go index 2f5b4d62..a1a884c4 100644 --- a/builder/static_code.go +++ b/builder/static_code.go @@ -1052,7 +1052,12 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { // ==template== {{ if not .Optimize }} var pt savepoint + // ==template== {{ if .LeftRecursion }} + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { + // {{ else }} if p.memoize { + // {{ end }} ==template== res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1065,7 +1070,11 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) // ==template== {{ if not .Optimize }} + // ==template== {{ if .LeftRecursion }} + if p.memoize && !isLeftRecusion { + // {{ else }} if p.memoize { + // {{ end }} ==template== p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } // {{ end }} ==template== diff --git a/test/issue_70b/issue_70b.go b/test/issue_70b/issue_70b.go index cb5b2c91..033621cb 100644 --- a/test/issue_70b/issue_70b.go +++ b/test/issue_70b/issue_70b.go @@ -1028,7 +1028,8 @@ func (p *parser) parseRule(rule *rule) (any, bool) { func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint - if p.memoize { + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1039,7 +1040,7 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) - if p.memoize { + if p.memoize && !isLeftRecusion { p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } return val, ok diff --git a/test/left_recursion/left_recursion_test.go b/test/left_recursion/left_recursion_test.go index 81885737..3bf41798 100644 --- a/test/left_recursion/left_recursion_test.go +++ b/test/left_recursion/left_recursion_test.go @@ -56,10 +56,70 @@ func TestLeftRecursionParse(t *testing.T) { for _, testCase := range tests { testCase := testCase - t.Run(testCase.name+" default", func(t *testing.T) { + + setOptionsLR := map[string][]leftrecursion.Option{ + "memoize": {leftrecursion.Memoize(true)}, + "-": {}, + } + for nameOptionsLR, optionsLR := range setOptionsLR { + optionsLR := optionsLR + + t.Run( + testCase.name+" default(recursion). Options: "+nameOptionsLR, + func(t *testing.T) { + t.Parallel() + + resLR, err := leftrecursion.Parse( + "", []byte(testCase.expr), optionsLR...) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + exprLR, ok := resLR.(string) + if !ok { + t.FailNow() + } + if exprLR != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, exprLR, testCase.want.expr) + } + }) + } + + setOptions := map[string][]withoutleftrecursion.Option{ + "memoize": {withoutleftrecursion.Memoize(true)}, + "-": {}, + } + for nameOptions, options := range setOptions { + options := options + + t.Run(testCase.name+" default(without recursion). Options: "+nameOptions, func(t *testing.T) { + t.Parallel() + + res, err := withoutleftrecursion.Parse("", []byte(testCase.expr), options...) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != testCase.want.expr { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + testCase.expr, expr, testCase.want.expr) + } + }) + } + + t.Run(testCase.name+" optimized(recursion)", func(t *testing.T) { t.Parallel() - resLR, err := leftrecursion.Parse("", []byte(testCase.expr)) + resLR, err := optimizedleftrecursion.Parse("", []byte(testCase.expr)) if err != nil { t.Fatalf( "for input %q got error: %s, but expect to parse without errors", @@ -74,41 +134,11 @@ func TestLeftRecursionParse(t *testing.T) { "for input %q\ngot result: %q,\nbut expect: %q", testCase.expr, exprLR, testCase.want.expr) } - res, err := withoutleftrecursion.Parse("", []byte(testCase.expr)) - if err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - testCase.expr, err) - } - expr, ok := res.(string) - if !ok { - t.FailNow() - } - if expr != testCase.want.expr { - t.Fatalf( - "for input %q\ngot result: %q,\nbut expect: %q", - testCase.expr, expr, testCase.want.expr) - } }) - t.Run(testCase.name+" optimized", func(t *testing.T) { + t.Run(testCase.name+" optimized(without recursion)", func(t *testing.T) { t.Parallel() - resLR, err := optimizedleftrecursion.Parse("", []byte(testCase.expr)) - if err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - testCase.expr, err) - } - exprLR, ok := resLR.(string) - if !ok { - t.FailNow() - } - if exprLR != testCase.want.expr { - t.Fatalf( - "for input %q\ngot result: %q,\nbut expect: %q", - testCase.expr, exprLR, testCase.want.expr) - } res, err := optimizedwithoutleftrecursion.Parse("", []byte(testCase.expr)) if err != nil { t.Fatalf( @@ -162,6 +192,43 @@ func FuzzLeftRecursionParse(f *testing.F) { }) } +func FuzzLeftRecursionParseMemoize(f *testing.F) { + chars := []byte("0123456789+-/*%") + + f.Fuzz(func(t *testing.T, bytes []byte) { + data := make([]byte, 0, len(bytes)) + for _, b := range bytes { + data = append(data, chars[int(b)%len(chars)]) + } + + resLR, errLR := leftrecursion.Parse( + "", data, leftrecursion.Memoize(true)) + res, err := withoutleftrecursion.Parse( + "", data, withoutleftrecursion.Memoize(true)) + if err != nil || errLR != nil { + if err == nil || errLR == nil { + t.Fatalf( + "for input %q\ngot error: %q,\nbut expect: %q", + data, errLR, err) + } + return + } + exprLR, okLR := resLR.(string) + if !okLR { + t.FailNow() + } + expr, ok := res.(string) + if !ok { + t.FailNow() + } + if expr != exprLR { + t.Fatalf( + "for input %q\ngot result: %q,\nbut expect: %q", + data, exprLR, expr) + } + }) +} + func FuzzLeftRecursionParseOptimized(f *testing.F) { chars := []byte("0123456789+-/*%") diff --git a/test/left_recursion/standart/leftrecursion/left_recursion.go b/test/left_recursion/standart/leftrecursion/left_recursion.go index d72b8bed..d7f60207 100644 --- a/test/left_recursion/standart/leftrecursion/left_recursion.go +++ b/test/left_recursion/standart/leftrecursion/left_recursion.go @@ -1294,7 +1294,8 @@ func (p *parser) parseRule(rule *rule) (any, bool) { func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint - if p.memoize { + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1305,7 +1306,7 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) - if p.memoize { + if p.memoize && !isLeftRecusion { p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } return val, ok diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go index 05330634..80e62906 100644 --- a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.go @@ -1304,7 +1304,8 @@ func (p *parser) parseRule(rule *rule) (any, bool) { func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint - if p.memoize { + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1315,7 +1316,7 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) - if p.memoize { + if p.memoize && !isLeftRecusion { p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } return val, ok diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go index 7889a05f..259c57ea 100644 --- a/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures_test.go @@ -103,54 +103,63 @@ func TestLeftRecursionWithLabeledFailures(t *testing.T) { } for _, testCase := range cases { testCase := testCase - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - got, err := leftrecursionlabeledfailures.Parse( - "", []byte(testCase.input)) - if testCase.want.errors == nil && err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - testCase.input, err) - } - if !reflect.DeepEqual(got, testCase.want.captures) { - t.Errorf( - "for input %q want %s, got %s", - testCase.input, testCase.want.captures, got) - } - if err != nil { - var errorLister leftrecursionlabeledfailures.ErrorLister - if !errors.As(err, &errorLister) { - t.FailNow() + setOptions := map[string][]leftrecursionlabeledfailures.Option{ + "memoize": {leftrecursionlabeledfailures.Memoize(true)}, + "-": {}, + } + for nameOptions, options := range setOptions { + options := options + + t.Run(testCase.name+". Options: "+nameOptions, func(t *testing.T) { + t.Parallel() + + got, err := leftrecursionlabeledfailures.Parse( + "", []byte(testCase.input), options...) + if testCase.want.errors == nil && err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.input, err) } - list := errorLister.Errors() - if len(list) != len(testCase.want.errors) { + if !reflect.DeepEqual(got, testCase.want.captures) { t.Errorf( - "for input %q want %d error(s), got %d", - testCase.input, len(testCase.want.errors), len(list)) - t.Logf("expected errors:\n") - for _, ee := range testCase.want.errors { - t.Logf("- %s\n", ee) - } - t.Logf("got errors:\n") - for _, ee := range list { - t.Logf("- %s\n", ee) - } - t.FailNow() + "for input %q want %s, got %s", + testCase.input, testCase.want.captures, got) } - for index, err := range list { - var parserError leftrecursionlabeledfailures.ParserError - if !errors.As(err, &parserError) { + if err != nil { + var errorLister leftrecursionlabeledfailures.ErrorLister + if !errors.As(err, &errorLister) { t.FailNow() } - if parserError.Error() != testCase.want.errors[index] { + list := errorLister.Errors() + if len(list) != len(testCase.want.errors) { t.Errorf( - "for input %q want %dth error to be %s, got %s", - testCase.input, index+1, - testCase.want.errors[index], parserError) + "for input %q want %d error(s), got %d", + testCase.input, len(testCase.want.errors), len(list)) + t.Logf("expected errors:\n") + for _, ee := range testCase.want.errors { + t.Logf("- %s\n", ee) + } + t.Logf("got errors:\n") + for _, ee := range list { + t.Logf("- %s\n", ee) + } + t.FailNow() + } + for index, err := range list { + var parserError leftrecursionlabeledfailures.ParserError + if !errors.As(err, &parserError) { + t.FailNow() + } + if parserError.Error() != testCase.want.errors[index] { + t.Errorf( + "for input %q want %dth error to be %s, got %s", + testCase.input, index+1, + testCase.want.errors[index], parserError) + } } } - } - }) + }) + } } } diff --git a/test/left_recursion_state/left_recursion_state_test.go b/test/left_recursion_state/left_recursion_state_test.go index db03d37e..f2149fe2 100644 --- a/test/left_recursion_state/left_recursion_state_test.go +++ b/test/left_recursion_state/left_recursion_state_test.go @@ -58,24 +58,36 @@ func TestLeftRecursionWithState(t *testing.T) { for _, testCase := range tests { testCase := testCase - t.Run(testCase.name+" default", func(t *testing.T) { - t.Parallel() - count, err := leftrecursionstate.Parse( - "", []byte(testCase.expr), - leftrecursionstate.Memoize(false), - leftrecursionstate.InitState("count", initCount)) - if err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - testCase.expr, err) - } - if count != testCase.want.count { - t.Fatalf( - "for input %q\ngot result: %d,\nbut expect: %d", - testCase.expr, count, testCase.want.count) - } - }) + setOptions := map[string][]leftrecursionstate.Option{ + "memoize": { + leftrecursionstate.Memoize(true), + leftrecursionstate.InitState("count", initCount), + }, + "-": { + leftrecursionstate.InitState("count", initCount), + }, + } + for nameOptions, options := range setOptions { + options := options + + t.Run(testCase.name+" default. Options: "+nameOptions, func(t *testing.T) { + t.Parallel() + + count, err := leftrecursionstate.Parse( + "", []byte(testCase.expr), options...) + if err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.expr, err) + } + if count != testCase.want.count { + t.Fatalf( + "for input %q\ngot result: %d,\nbut expect: %d", + testCase.expr, count, testCase.want.count) + } + }) + } t.Run(testCase.name+" optimized", func(t *testing.T) { t.Parallel() diff --git a/test/left_recursion_state/standart/left_recursion_state.go b/test/left_recursion_state/standart/left_recursion_state.go index cc315ac9..78aefa1b 100644 --- a/test/left_recursion_state/standart/left_recursion_state.go +++ b/test/left_recursion_state/standart/left_recursion_state.go @@ -1314,7 +1314,8 @@ func (p *parser) parseRule(rule *rule) (any, bool) { func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint - if p.memoize { + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -1325,7 +1326,7 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) - if p.memoize { + if p.memoize && !isLeftRecusion { p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } return val, ok diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go index 5bb34da5..624cd991 100644 --- a/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover.go @@ -2137,7 +2137,8 @@ func (p *parser) parseRule(rule *rule) (any, bool) { func (p *parser) parseExprWrap(expr any) (any, bool) { var pt savepoint - if p.memoize { + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { res, ok := p.getMemoized(expr) if ok { p.restore(res.end) @@ -2148,7 +2149,7 @@ func (p *parser) parseExprWrap(expr any) (any, bool) { val, ok := p.parseExpr(expr) - if p.memoize { + if p.memoize && !isLeftRecusion { p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) } return val, ok diff --git a/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go index 1cbe0bc8..e141f791 100644 --- a/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go +++ b/test/left_recursion_thrownrecover/left_recursion_thrownrecover_test.go @@ -149,59 +149,73 @@ func TestLeftRecursionWithThrowAndRecover(t *testing.T) { } for _, testCase := range cases { testCase := testCase - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - got, err := leftrecursionthrownrecover.Parse( - "", []byte(testCase.input), - leftrecursionthrownrecover.Entrypoint(testCase.entrypoint)) - if testCase.want.errors == nil && err != nil { - t.Fatalf( - "for input %q got error: %s, but expect to parse without errors", - testCase.input, err) - } - if testCase.want.errors != nil && err == nil { - t.Fatalf( - "for input %q got no error, but expect to parse with errors: %s", - testCase.input, testCase.want.errors) - } - if !reflect.DeepEqual(got, testCase.want.captures) { - t.Errorf( - "for input %q want %s, got %s", - testCase.input, testCase.want.captures, got) - } - if err != nil { - var errorLister leftrecursionthrownrecover.ErrorLister - if !errors.As(err, &errorLister) { - t.FailNow() + + setOptions := map[string][]leftrecursionthrownrecover.Option{ + "memoize": { + leftrecursionthrownrecover.Memoize(true), + leftrecursionthrownrecover.Entrypoint(testCase.entrypoint), + }, + "-": { + leftrecursionthrownrecover.Entrypoint(testCase.entrypoint), + }, + } + for nameOptions, options := range setOptions { + options := options + + t.Run(testCase.name+". Options: "+nameOptions, func(t *testing.T) { + t.Parallel() + + got, err := leftrecursionthrownrecover.Parse( + "", []byte(testCase.input), options...) + if testCase.want.errors == nil && err != nil { + t.Fatalf( + "for input %q got error: %s, but expect to parse without errors", + testCase.input, err) } - list := errorLister.Errors() - if len(list) != len(testCase.want.errors) { + if testCase.want.errors != nil && err == nil { + t.Fatalf( + "for input %q got no error, but expect to parse with errors: %s", + testCase.input, testCase.want.errors) + } + if !reflect.DeepEqual(got, testCase.want.captures) { t.Errorf( - "for input %q want %d error(s), got %d", - testCase.input, len(testCase.want.errors), len(list)) - t.Logf("expected errors:\n") - for _, ee := range testCase.want.errors { - t.Logf("- %s\n", ee) - } - t.Logf("got errors:\n") - for _, ee := range list { - t.Logf("- %s\n", ee) - } - t.FailNow() + "for input %q want %s, got %s", + testCase.input, testCase.want.captures, got) } - for index, err := range list { - var parserError leftrecursionthrownrecover.ParserError - if !errors.As(err, &parserError) { + if err != nil { + var errorLister leftrecursionthrownrecover.ErrorLister + if !errors.As(err, &errorLister) { t.FailNow() } - if parserError.Error() != testCase.want.errors[index] { + list := errorLister.Errors() + if len(list) != len(testCase.want.errors) { t.Errorf( - "for input %q want %dth error to be %s, got %s", - testCase.input, index+1, - testCase.want.errors[index], parserError) + "for input %q want %d error(s), got %d", + testCase.input, len(testCase.want.errors), len(list)) + t.Logf("expected errors:\n") + for _, ee := range testCase.want.errors { + t.Logf("- %s\n", ee) + } + t.Logf("got errors:\n") + for _, ee := range list { + t.Logf("- %s\n", ee) + } + t.FailNow() + } + for index, err := range list { + var parserError leftrecursionthrownrecover.ParserError + if !errors.As(err, &parserError) { + t.FailNow() + } + if parserError.Error() != testCase.want.errors[index] { + t.Errorf( + "for input %q want %dth error to be %s, got %s", + testCase.input, index+1, + testCase.want.errors[index], parserError) + } } } - } - }) + }) + } } } From 1faea726cf3bdb17328c51c5f7f948a915c88000 Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Wed, 30 Aug 2023 21:01:13 +0200 Subject: [PATCH 13/14] Fix formatting --- test/left_recursion/left_recursion.peg | 2 +- .../left_recursion_labeled_failures.peg | 2 +- test/left_recursion_state/left_recursion_state.peg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/left_recursion/left_recursion.peg b/test/left_recursion/left_recursion.peg index 895b521c..fc207805 100644 --- a/test/left_recursion/left_recursion.peg +++ b/test/left_recursion/left_recursion.peg @@ -35,4 +35,4 @@ factor = op:('+' / '-') a:factor { return string(c.text), nil } -atom = [0-9]+ \ No newline at end of file +atom = [0-9]+ diff --git a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg index d0b3181c..dc43fda0 100644 --- a/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg +++ b/test/left_recursion_labeled_failures/left_recursion_labeled_failures.peg @@ -37,4 +37,4 @@ ErrComma ← #{ } ( !([a-z]+) .)* ErrID ← #{ return errors.New("expecting an identifier") - } ( !(',') .)* { return "NONE", nil } \ No newline at end of file + } ( !(',') .)* { return "NONE", nil } diff --git a/test/left_recursion_state/left_recursion_state.peg b/test/left_recursion_state/left_recursion_state.peg index 679dc2ce..5eedd8e4 100644 --- a/test/left_recursion_state/left_recursion_state.peg +++ b/test/left_recursion_state/left_recursion_state.peg @@ -38,4 +38,4 @@ factor = (('+' / '-') factor) #{ atom = ([0-9]+) #{ c.state["count"] = c.state["count"].(int) + 127; return nil -} \ No newline at end of file +} From e5cbadb649371d2a340e28486bbffb9f4e81bb1f Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Wed, 30 Aug 2023 21:01:30 +0200 Subject: [PATCH 14/14] Add test case for issue #79 --- Makefile | 4 + test/issue_79/issue_79.go | 1591 ++++++++++++++++++++++++++++++++++++ test/issue_79/issue_79.peg | 21 + 3 files changed, 1616 insertions(+) create mode 100644 test/issue_79/issue_79.go create mode 100644 test/issue_79/issue_79.peg diff --git a/Makefile b/Makefile index 932d3281..2e517347 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,10 @@ $(TEST_DIR)/issue_70/optimized-grammar/issue_70.go: $(TEST_DIR)/issue_70/issue_7 $(TEST_DIR)/issue_70b/issue_70b.go: $(TEST_DIR)/issue_70b/issue_70b.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint -optimize-grammar -support-left-recursion $< > $@ +$(TEST_DIR)/issue_79/issue_79.go: $(TEST_DIR)/issue_79/issue_79.peg $(BINDIR)/pigeon + @! $(BINDIR)/pigeon $< > $@ 2>/dev/null && exit 0 || echo "failure, expect build to fail due to left recursion!" && exit 1 + $(BINDIR)/pigeon -support-left-recursion $< > $@ + $(TEST_DIR)/issue_80/issue_80.go: $(TEST_DIR)/issue_80/issue_80.peg $(BINDIR)/pigeon $(BINDIR)/pigeon -nolint $< > $@ diff --git a/test/issue_79/issue_79.go b/test/issue_79/issue_79.go new file mode 100644 index 00000000..3d9f8c03 --- /dev/null +++ b/test/issue_79/issue_79.go @@ -0,0 +1,1591 @@ +// Code generated by pigeon; DO NOT EDIT. + +package issue79 + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "Input", + pos: position{line: 5, col: 1, offset: 22}, + expr: &actionExpr{ + pos: position{line: 5, col: 10, offset: 31}, + run: (*parser).callonInput1, + expr: &seqExpr{ + pos: position{line: 5, col: 10, offset: 31}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 5, col: 10, offset: 31}, + label: "expr", + expr: &ruleRefExpr{ + pos: position{line: 5, col: 15, offset: 36}, + name: "Expr", + }, + }, + &ruleRefExpr{ + pos: position{line: 5, col: 20, offset: 41}, + name: "EOF", + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "Expr", + pos: position{line: 9, col: 1, offset: 68}, + expr: &choiceExpr{ + pos: position{line: 9, col: 9, offset: 76}, + alternatives: []any{ + &seqExpr{ + pos: position{line: 9, col: 9, offset: 76}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 9, col: 9, offset: 76}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 11, offset: 78}, + name: "Expr", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 16, offset: 83}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 18, offset: 85}, + name: "LogicOp", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 26, offset: 93}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 28, offset: 95}, + name: "Expr", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 33, offset: 100}, + name: "_", + }, + }, + }, + &seqExpr{ + pos: position{line: 9, col: 36, offset: 103}, + exprs: []any{ + &ruleRefExpr{ + pos: position{line: 9, col: 36, offset: 103}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 38, offset: 105}, + name: "Value", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 44, offset: 111}, + name: "_", + }, + }, + }, + }, + }, + leader: true, + leftRecursive: true, + }, + { + name: "LogicOp", + pos: position{line: 11, col: 1, offset: 114}, + expr: &actionExpr{ + pos: position{line: 11, col: 12, offset: 125}, + run: (*parser).callonLogicOp1, + expr: &choiceExpr{ + pos: position{line: 11, col: 13, offset: 126}, + alternatives: []any{ + &litMatcher{ + pos: position{line: 11, col: 13, offset: 126}, + val: "and", + ignoreCase: false, + want: "\"and\"", + }, + &litMatcher{ + pos: position{line: 11, col: 21, offset: 134}, + val: "or", + ignoreCase: false, + want: "\"or\"", + }, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "Value", + pos: position{line: 15, col: 1, offset: 173}, + expr: &actionExpr{ + pos: position{line: 15, col: 10, offset: 182}, + run: (*parser).callonValue1, + expr: &oneOrMoreExpr{ + pos: position{line: 15, col: 10, offset: 182}, + expr: &charClassMatcher{ + pos: position{line: 15, col: 10, offset: 182}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "_", + displayName: "\"whitespace\"", + pos: position{line: 19, col: 1, offset: 221}, + expr: &zeroOrMoreExpr{ + pos: position{line: 19, col: 19, offset: 239}, + expr: &charClassMatcher{ + pos: position{line: 19, col: 19, offset: 239}, + val: "[ \\n\\t\\r]", + chars: []rune{' ', '\n', '\t', '\r'}, + ignoreCase: false, + inverted: false, + }, + }, + leader: false, + leftRecursive: false, + }, + { + name: "EOF", + pos: position{line: 21, col: 1, offset: 251}, + expr: ¬Expr{ + pos: position{line: 21, col: 8, offset: 258}, + expr: &anyMatcher{ + line: 21, col: 9, offset: 259, + }, + }, + leader: false, + leftRecursive: false, + }, + }, +} + +func (c *current) onInput1(expr any) (any, error) { + return expr, nil +} + +func (p *parser) callonInput1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onInput1(stack["expr"]) +} + +func (c *current) onLogicOp1() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonLogicOp1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLogicOp1() +} + +func (c *current) onValue1() (any, error) { + return string(c.text), nil +} + +func (p *parser) callonValue1() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onValue1() +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value any) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i any, err error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (any, error) { + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (any, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]any + +// the AST types... + +type grammar struct { + pos position + rules []*rule +} + +type rule struct { + pos position + name string + displayName string + expr any + + leader bool + leftRecursive bool +} + +type choiceExpr struct { + pos position + alternatives []any +} + +type actionExpr struct { + pos position + expr any + run func(*parser) (any, error) +} + +type recoveryExpr struct { + pos position + expr any + recoverExpr any + failureLabel []string +} + +type seqExpr struct { + pos position + exprs []any +} + +type throwExpr struct { + pos position + label string +} + +type labeledExpr struct { + pos position + label string + expr any +} + +type expr struct { + pos position + expr any +} + +type ( + andExpr expr + notExpr expr + zeroOrOneExpr expr + zeroOrMoreExpr expr + oneOrMoreExpr expr +) + +type ruleRefExpr struct { + pos position + name string +} + +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type litMatcher struct { + pos position + val string + ignoreCase bool + want string +} + +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +type resultTuple struct { + v any + b bool + end savepoint +} + +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type ruleWithExpsStack struct { + rule *rule + estack []any +} + +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[any]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]any + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]any +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]any) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr any) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]any, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) printIndent(mark string, s string) string { + return p.print(strings.Repeat(" ", p.depth)+mark, s) +} + +func (p *parser) in(s string) string { + res := p.printIndent(">", s) + p.depth++ + return res +} + +func (p *parser) out(s string) string { + p.depth-- + return p.printIndent("<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() any +} + +var statePool = &sync.Pool{ + New: func() any { return make(storeDict) }, +} + +func (sd storeDict) Discard() { + for k := range sd { + delete(sd, k) + } + statePool.Put(sd) +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := statePool.Get().(storeDict) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state.Discard() + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node any) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node any, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[any]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[any]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +func (p *parser) parse(g *grammar) (val any, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRuleWrap(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] + } +} + +func (p *parser) parseRuleRecursiveLeader(rule *rule) (any, bool) { + result, ok := p.getMemoized(rule) + if ok { + p.restore(result.end) + return result.v, result.b + } + + if p.debug { + defer p.out(p.in("recursive " + rule.name)) + } + + var ( + depth = 0 + startMark = p.pt + lastResult = resultTuple{nil, false, startMark} + lastErrors = *p.errs + ) + + for { + lastState := p.cloneState() + p.setMemoized(startMark, rule, lastResult) + val, ok := p.parseRule(rule) + endMark := p.pt + if p.debug { + p.printIndent("RECURSIVE", fmt.Sprintf( + "Rule %s depth %d: %t -> %s", + rule.name, depth, ok, string(p.sliceFrom(startMark)))) + } + if (!ok) || (endMark.offset <= lastResult.end.offset && depth != 0) { + p.restoreState(lastState) + *p.errs = lastErrors + break + } + lastResult = resultTuple{val, ok, endMark} + lastErrors = *p.errs + p.restore(startMark) + depth++ + } + + p.restore(lastResult.end) + p.setMemoized(startMark, rule, lastResult) + return lastResult.v, lastResult.b +} + +func (p *parser) parseRuleRecursiveNoLeader(rule *rule) (any, bool) { + return p.parseRule(rule) +} + +func (p *parser) parseRuleMemoize(rule *rule) (any, bool) { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + + startMark := p.pt + val, ok := p.parseRule(rule) + p.setMemoized(startMark, rule, resultTuple{val, ok, p.pt}) + + return val, ok +} + +func (p *parser) parseRuleWrap(rule *rule) (any, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + var ( + val any + ok bool + startMark = p.pt + ) + + if p.memoize || rule.leftRecursive { + if rule.leader { + val, ok = p.parseRuleRecursiveLeader(rule) + } else if p.memoize && !rule.leftRecursive { + val, ok = p.parseRuleMemoize(rule) + } else { + val, ok = p.parseRuleRecursiveNoLeader(rule) + } + } else { + val, ok = p.parseRule(rule) + } + + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(startMark))) + } + return val, ok +} + +func (p *parser) parseRule(rule *rule) (any, bool) { + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExprWrap(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + return val, ok +} + +func (p *parser) parseExprWrap(expr any) (any, bool) { + var pt savepoint + + isLeftRecusion := p.rstack[len(p.rstack)-1].leftRecursive + if p.memoize && !isLeftRecusion { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + val, ok := p.parseExpr(expr) + + if p.memoize && !isLeftRecusion { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseExpr(expr any) (any, bool) { + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val any + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExprWrap(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.printIndent("MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExprWrap(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExprWrap(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExprWrap(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, lit.want) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, lit.want) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExprWrap(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExprWrap(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRuleWrap(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]any, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExprWrap(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExprWrap(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []any + + for { + p.pushV() + val, ok := p.parseExprWrap(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExprWrap(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/test/issue_79/issue_79.peg b/test/issue_79/issue_79.peg new file mode 100644 index 00000000..1e0b289a --- /dev/null +++ b/test/issue_79/issue_79.peg @@ -0,0 +1,21 @@ +{ + package issue79 +} + +Input <- expr:Expr EOF { + return expr, nil +} + +Expr <- _ Expr _ LogicOp _ Expr _/ _ Value _ + +LogicOp <- ("and" / "or") { + return string(c.text), nil +} + +Value <- [0-9]+ { + return string(c.text),nil +} + +_ "whitespace" <- [ \n\t\r]* + +EOF <- !.