Support optional for all symbols (#144)

src/main/resources/manuals/d711ba960cd12b76/tycheck-ignore.json

@@ -33,7 +33,6 @@
  "GetSubstitution",
  "GetThisValue",
  "GetValue",
- "HostEnqueuePromiseJob",
  "InnerModuleEvaluation",
  "IntegerIndexedObjectCreate",
  "LabelledItem[1,0].LabelledEvaluation",
@@ -62,4 +61,4 @@
  "StringCreate",
  "ToBigInt",
  "ToNumber"
-]
+]

src/main/scala/esmeta/analyzer/AbsTransfer.scala

@@ -164,7 +164,7 @@ class AbsTransfer(sem: AbsSemantics) extends Optimized with PruneHelper {
  case Syntactic(name, _, rhsIdx, children) =>
  val rhs = cfg.grammar.nameMap(name).rhsList(rhsIdx)
  val optionals = (for {
- (opt, child) <- rhs.nts.map(_.optional) zip children if opt
+ ((_, opt), child) <- rhs.ntsWithOptional zip children if opt
  } yield !child.isEmpty)
  optionals.reverse.zipWithIndex.foldLeft(0) {
  case (acc, (true, idx)) => acc + scala.math.pow(2, idx).toInt

src/main/scala/esmeta/analyzer/domain/value/TypeDomain.scala

@@ -510,7 +510,7 @@ object TypeDomain extends value.Domain {
  case Production(lhs, _, _, rhsList) =>
  val name = lhs.name
  val subs = rhsList.collect {
- case Rhs(_, List(Nonterminal(name, _, _)), _) => name
+ case Rhs(_, List(Nonterminal(name, _)), _) => name
  }.toSet
  name -> subs
  }).toMap

src/main/scala/esmeta/interpreter/Interpreter.scala

@@ -610,7 +610,7 @@ class Interpreter(
  case Syntactic(name, _, rhsIdx, children) =>
  val rhs = cfg.grammar.nameMap(name).rhsList(rhsIdx)
  val optionals = (for {
- (opt, child) <- rhs.nts.map(_.optional) zip children if opt
+ ((_, opt), child) <- rhs.ntsWithOptional zip children if opt
  } yield !child.isEmpty)
  optionals.reverse.zipWithIndex.foldLeft(0) {
  case (acc, (true, idx)) => acc + scala.math.pow(2, idx).toInt

src/main/scala/esmeta/mutator/synthesizer/RandomSynthesizer.scala

@@ -36,10 +36,11 @@ class RandomSynthesizer(
  symbol: Symbol,
  ): Option[Option[Ast]] = symbol match
  case ButNot(nt, _) => synSymbol(argsMap)(nt)
- case Nonterminal(name, args, optional) =>
+ case Optional(symbol) =>
+ if (randBool) Some(None) else synSymbol(argsMap)(symbol)
+ case Nonterminal(name, args) =>
  if (reservedLexicals contains name)
  Some(Some(Lexical(name, reservedLexicals(name))))
- else if (optional && randBool) Some(None)
  else {
  import NonterminalArgumentKind.*
  val newArgs = for (arg <- args) yield arg.kind match

src/main/scala/esmeta/mutator/synthesizer/SimpleSynthesizer.scala

@@ -42,11 +42,11 @@ class SimpleSynthesizer(
  private def synSymbol(argsMap: Map[String, Boolean])(
  symbol: Symbol,
  ): Option[Option[Ast]] = symbol match
- case ButNot(nt, _) => synSymbol(argsMap)(nt)
- case Nonterminal(name, args, optional) =>
+ case ButNot(nt, _) => synSymbol(argsMap)(nt)
+ case Optional(symbol) => Some(None)
+ case Nonterminal(name, args) =>
  if (reservedLexicals contains name)
  Some(Some(Lexical(name, reservedLexicals(name))))
- else if (optional) Some(None)
  else {
  import NonterminalArgumentKind.*
  val newArgs = for (arg <- args) yield arg.kind match

src/main/scala/esmeta/parser/ESParser.scala

@@ -148,37 +148,37 @@ case class ESParser(
  prev: LAParser[List[Option[Ast]]],
  symbol: Symbol,
  argsSet: Set[String],
- ): LAParser[List[Option[Ast]]] =
- symbol match {
- case Terminal(")") if name == "DoWhileStatement" => prev <~ doWhileCloseT
- case Terminal(term) => prev <~ t(term)
- case Nonterminal(name, args, optional) =>
- lazy val parser =
+ optional: Boolean = false,
+ ): LAParser[List[Option[Ast]]] = symbol match
+ case Terminal(")") if name == "DoWhileStatement" => prev <~ doWhileCloseT
+ case Terminal(term) => prev <~ t(term)
+ case symbol: NtBase =>
+ lazy val parser = symbol match
+ case Nonterminal(name, args) =>
  if (lexNames contains name) nt(name, lexers(name, 0))
  else parsers(name)(toBools(argsSet, args))
- if (optional) prev ~ opt(parser) ^^ { case l ~ s => s :: l }
- else prev ~ parser ^^ { case l ~ s => Some(s) :: l }
- case ButNot(base, cases) =>
- val name = s"$base \\ ${cases.mkString("(", ", ", ")")}"
- lazy val parser = nt(name, getSymbolParser(symbol, argsSet))
- prev ~ parser ^^ { case l ~ s => Some(s) :: l }
- case ButOnlyIf(base, methodName, cond) => ???
- case Lookahead(contains, cases) =>
- lazy val parser = cases
- .map(
- _.map(_ match {
- case Terminal(t) if t.matches("[a-z]+") => t <~ not(IDContinue)
- case symbol => getSymbolParser(symbol, argsSet)
- }).reduce(_ %% _),
- )
- .reduce(_ | _)
- lazy val lookahead = ntl(symbol.toString, parser)
- prev <~ (if (contains) +lookahead else -lookahead)
- case Empty => prev
- case NoLineTerminator => prev <~ noLineTerminator
- case CodePointAbbr(abbr) => ???
- case UnicodeSet(cond) => ???
- }
+ case ButNot(base, cases) =>
+ val name = s"$base \\ ${cases.mkString("(", ", ", ")")}"
+ nt(name, getSymbolParser(symbol, argsSet))
+ case ButOnlyIf(base, methodName, cond) => ???
+ if (optional) prev ~ opt(parser) ^^ { case l ~ s => s :: l }
+ else prev ~ parser ^^ { case l ~ s => Some(s) :: l }
+ case Optional(symbol) => appendParser(name, prev, symbol, argsSet, true)
+ case Lookahead(contains, cases) =>
+ lazy val parser = cases
+ .map(
+ _.map(_ match {
+ case Terminal(t) if t.matches("[a-z]+") => t <~ not(IDContinue)
+ case symbol => getSymbolParser(symbol, argsSet)
+ }).reduce(_ %% _),
+ )
+ .reduce(_ | _)
+ lazy val lookahead = ntl(symbol.toString, parser)
+ prev <~ (if (contains) +lookahead else -lookahead)
+ case Empty => prev
+ case NoLineTerminator => prev <~ noLineTerminator
+ case CodePointAbbr(abbr) => ???
+ case UnicodeSet(cond) => ???
 
  // a terminal lexer
  protected val TERMINAL: Lexer = grammar.prods.foldLeft[Parser[String]]("") {
@@ -428,10 +428,10 @@ case class ESParser(
  }
 
  // check whether it is a left-recursive production
- private def isLR(name: String, rhs: Rhs): Boolean = rhs.symbols match {
- case Nonterminal(`name`, _, _) :: _ => true
- case _ => false
- }
+ private def isLR(name: String, rhs: Rhs): Boolean = (for {
+ symbol <- rhs.symbols.headOption
+ nt <- symbol.getNt
+ } yield nt.name == name).getOrElse(false)
 
  // TODO more general rule
  // handle indirect left-recursive case

src/main/scala/esmeta/parser/Lexer.scala

@@ -94,10 +94,10 @@ trait Lexer extends UnicodeParsers {
  argsSet: Set[String],
  ): Parser[String] = symbol match
  case Terminal(term) => term
- case Nonterminal(name, args, optional) =>
- val parser = lexers((name, toBit(toBools(argsSet, args))))
-  if (optional) opt(parser) ^^ { _.getOrElse("") }
- else parser
+ case Nonterminal(name, args) =>
+ lexers((name, toBit(toBools(argsSet, args))))
+ case Optional(symbol) =>
+ opt(getSymbolParser(symbol, argsSet)) ^^ { _.getOrElse("") }
  case ButNot(base, cases) =>
  val parser = getSymbolParser(base, argsSet)
  val exclude = cases.map(getSymbolParser(_, argsSet)).reduce(_ ||| _)

src/main/scala/esmeta/spec/Production.scala

@@ -19,12 +19,18 @@ case class Production(
  (name, j) <- rhs.allNames.zipWithIndex
  } yield lhs.name + ":" + name -> (i, j)).toMap
 
- /** get non-terminals in RHSs */
+ /** get non-terminals with whether it is optional in an RHS */
  lazy val nts: List[Nonterminal] = for {
  rhs <- rhsList
  nt <- rhs.nts
  } yield nt
 
+ /** get non-terminals with whether it is optional in an RHS */
+ lazy val ntsWithOptional: List[(Nonterminal, Boolean)] = for {
+ rhs <- rhsList
+ pair <- rhs.ntsWithOptional
+ } yield pair
+
  /** get terminals in RHSs */
  lazy val ts: List[Terminal] = for {
  rhs <- rhsList

src/main/scala/esmeta/spec/Rhs.scala

@@ -15,36 +15,49 @@ case class Rhs(
  /** get RHS all names */
  def allNames: List[String] =
  symbols.foldLeft(List[String]("")) {
- case (names, Terminal(term)) => names.map(_ + term)
- case (names, Nonterminal(name, _, optional)) =>
- names.flatMap(x => {
- if (optional) List(x, x + name) else List(x + name)
- })
- case (names, ButNot(base, _)) =>
- names.map(_ + base.name)
- case (names, ButOnlyIf(base, _, _)) =>
- names.map(_ + base.name)
- case (names, _) => names
+ case (names, Terminal(term)) =>
+ names.map(_ + term)
+ case (names, s: Optional) =>
+ s.getName.fold(names)(name => names.flatMap(x => List(x, x + name)))
+ case (names, s) =>
+ s.getName.fold(names)(name => names.map(_ + name))
  }
 
- /** get non-terminals in an RHS */
- lazy val nts: List[Nonterminal] = symbols.flatMap(_.getNt)
+ /** optional symbols */
+ lazy val optionals = symbols.collect { case opt: Optional => opt }
+
+ lazy val nts: List[Nonterminal] = for {
+ symbol <- symbols
+ nt <- symbol.getNt
+ } yield nt
+
+ /** get non-terminals with whether it is optional in an RHS */
+ lazy val ntsWithOptional: List[(Nonterminal, Boolean)] = for {
+ symbol <- symbols
+ nt <- symbol.getNt
+ } yield (nt, symbol.isInstanceOf[Optional])
+
+ /** get terminals in an RHS */
  lazy val ts: List[Terminal] = symbols.flatMap(_.getT)
+
+ /** get terminals in an RHS */
  lazy val getNts = cached[Int, List[Option[String]]] { subIdx =>
  val binStr = subIdx.toBinaryString
- val optCount = nts.count(_.optional)
+ val optCount = optionals.size
  var flags = (("0" * (optCount - binStr.length)) + binStr).map(_ == '1')
- nts.map(nt =>
- if (nt.optional) {
+ for {
+ symbol <- symbols
+ nt <- symbol.getNt
+ } yield symbol match
+ case _: Optional =>
  val present = flags.head
  flags = flags.tail
  if (present) Some(nt.name) else None
- } else Some(nt.name),
- )
+ case _ => Some(nt.name)
  }
 
  /** count sub production */
- lazy val countSubs: Int = scala.math.pow(2, nts.count(_.optional)).toInt
+ lazy val countSubs: Int = scala.math.pow(2, optionals.size).toInt
 
  /** check if empty */
  def isEmpty: Boolean = symbols match
@@ -53,14 +66,10 @@ case class Rhs(
 
  /** get index of non-terminal */
  def getNtIndex(ntName: String): Option[Int] =
- val filtered = nts.zipWithIndex.filter { case (nt, _) => nt.name == ntName }
- filtered match
- case (_, idx) :: rest => Some(idx)
- case _ => None
+ nts.zipWithIndex.find(_.head.name == ntName).map(_.last)
 
  /** get parameters from RHSs */
- def params: List[Param] =
- nts.map(nt => Param(nt.name, Type(AstT(nt.name))))
+ def params: List[Param] = nts.map(nt => Param(nt.name, Type(AstT(nt.name))))
 
  /** check whether the RHS is available */
  def available(argsSet: Set[String]): Boolean = conditions.forall {

src/main/scala/esmeta/spec/Symbol.scala

@@ -8,26 +8,36 @@ sealed trait Symbol extends SpecElem {
  /** get an non-terminal or nothing from a symbol */
  def getNt: Option[Nonterminal] = this match
  case (nt: Nonterminal) => Some(nt)
+ case Optional(symbol) => symbol.getNt
  case ButNot(base, _) => Some(base)
  case ButOnlyIf(base, _, _) => Some(base)
  case _ => None
 
  /** get a terminal or nothing from a symbol */
- def getT: Option[Terminal] = this match {
+ def getT: Option[Terminal] = this match
  case t: Terminal => Some(t)
  case _ => None
- }
+
+ /** get the name of a symbol */
+ def getName: Option[String] = this match
+ case Nonterminal(name, _) => Some(name)
+ case Optional(symbol) => symbol.getName
+ case ButNot(base, _) => Some(base.name)
+ case ButOnlyIf(base, _, _) => Some(base.name)
+ case _ => None
 }
 object Symbol extends Parser.From(Parser.symbol)
 
 /** terminal symbols */
 case class Terminal(term: String) extends Symbol
 
+/** nonterminal symbols as base symbols */
+type NtBase = Nonterminal | ButNot | ButOnlyIf
+
 /** nonterminal symbols */
 case class Nonterminal(
  name: String,
  args: List[NonterminalArgument],
- optional: Boolean,
 ) extends Symbol
 case class NonterminalArgument(
  kind: NonterminalArgumentKind,
@@ -37,6 +47,9 @@ object NonterminalArgument extends Parser.From(Parser.ntArg)
 enum NonterminalArgumentKind extends SpecElem { case True, False, Pass }
 object NonterminalArgumentKind extends Parser.From(Parser.ntArgKind)
 
+/** optional symbols */
+case class Optional(symbol: Symbol) extends Symbol
+
 /** butnot symbols */
 case class ButNot(base: Nonterminal, notCases: List[Symbol]) extends Symbol
 

src/main/scala/esmeta/spec/util/JsonProtocol.scala

@@ -44,6 +44,7 @@ object JsonProtocol extends BasicJsonProtocol {
  : List[(String, HCursor => Decoder.Result[Symbol])] = List(
  "term" -> (_.as[Terminal]),
  "name" -> (_.as[Nonterminal]),
+ "symbol" -> (_.as[Optional]),
  "notCases" -> (_.as[ButNot]),
  "methodName" -> (_.as[ButOnlyIf]),
  "cases" -> (_.as[Lookahead]),
@@ -55,6 +56,7 @@ object JsonProtocol extends BasicJsonProtocol {
  given Encoder[Symbol] = Encoder.instance {
  case symbol: Terminal => symbol.asJson
  case symbol: Nonterminal => symbol.asJson
+ case symbol: Optional => symbol.asJson
  case symbol: ButNot => symbol.asJson
  case symbol: ButOnlyIf => symbol.asJson
  case symbol: Lookahead => symbol.asJson
@@ -71,6 +73,8 @@ object JsonProtocol extends BasicJsonProtocol {
  given Encoder[NonterminalArgument] = deriveEncoder
  given Decoder[NonterminalArgumentKind] = deriveDecoder
  given Encoder[NonterminalArgumentKind] = deriveEncoder
+ given Decoder[Optional] = deriveDecoder
+ given Encoder[Optional] = deriveEncoder
  given Decoder[ButNot] = deriveDecoder
  given Encoder[ButNot] = deriveEncoder
  given Decoder[ButOnlyIf] = deriveDecoder

src/main/scala/esmeta/spec/util/Parser.scala

@@ -80,7 +80,10 @@ trait Parsers extends LangParsers {
 
  /** grammar symbols */
  given symbol: Parser[Symbol] = {
- term | butnot | lookahead | butOnlyIf | nt | abbr | unicodeSet | empty | nlt
+ {
+ term | butnot | lookahead | butOnlyIf | nt | abbr | unicodeSet |
+ empty | nlt
+ } ~ opt("?") ^^ { case s ~ o => if (o.isDefined) Optional(s) else s }
  }.named("spec.Symbol")
 
  /** terminals */
@@ -93,9 +96,8 @@ trait Parsers extends LangParsers {
 
  /** nonterminals */
  lazy val nt: Parser[Nonterminal] = {
- word ~ opt("[" ~> rep1sep(ntArg, ",") <~ "]") ~ opt("?") ^^ {
- case name ~ args ~ opt =>
- Nonterminal(name, args.getOrElse(Nil), opt.isDefined)
+ word ~ opt("[" ~> rep1sep(ntArg, ",") <~ "]") ^^ {
+ case name ~ args => Nonterminal(name, args.getOrElse(Nil))
  }
  }.named("spec.Nonterminal")
 

src/main/scala/esmeta/spec/util/Stringifier.scala

@@ -129,10 +129,11 @@ object Stringifier {
  case Empty => app >> "[empty]"
  case NoLineTerminator => app >> "[no LineTerminator here]"
  case CodePointAbbr(abbr) => app >> "<" >> abbr >> ">"
- case Nonterminal(name, args, opt) =>
+ case Nonterminal(name, args) =>
  app >> name
- if (!args.isEmpty) app >> args
- if (opt) app >> "?" else app
+ if (!args.isEmpty) app >> args else app
+ case Optional(symbol) =>
+ app >> symbol >> "?"
  case Lookahead(b, cases) =>
  app >> "[lookahead " >> (if (b) "<" else "<!") >> " " >> cases >> "]"
  case ButOnlyIf(base, name, cond) =>