Improve repeat and non-greedy handling

Author: ceedubs

benchmarks/src/main/scala/ZeroStarStarABenchmarks.scala

@@ -12,7 +12,7 @@ import org.openjdk.jmh.annotations.{Benchmark, Scope, State}
 class ZeroStarStarABenchmarks {
   val manyZeros: String = "0" * 20
   val java: Pattern = Pattern.compile("(0*)*A")
-  val irrec: RegexC[Unit] = lit('0').chain(Greedy).chain(Greedy).void <* lit('A')
+  val irrec: RegexC[Unit] = lit('0').star(Greedy).star(Greedy).void <* lit('A')
   val irrecMatcher: String => Boolean = irrec.stringMatcher
 
   @Benchmark

docs/usage.md

@@ -89,7 +89,7 @@ duration.toPattern
 
 ### random matches for a regular expression
 
-Irrec provides support for creating [Scalacheck](https://www.scalacheck.org/) generators that produce values that match a regular expression. This generation is done efficiently as opposed to generating a bunch of random values and then filtering the ones that don't match the regular expression (which would quickly lead to Scalacheck giving up on generating matching values).
+Irrec provides [Scalacheck](https://www.scalacheck.org/) generators that produce values that match a regular expression. These can be useful for tests, or even just for glancing at random matches to ensure that your regular expression does what you intended. Check out the [regex-explorer](regex-explorer.md) for interactive browser-based regular expression exploration powered by irrec.
 
 ```scala mdoc:silent
 import ceedubs.irrec.regex.gen._, CharRegexGen._
@@ -103,20 +103,24 @@ val genDurationString: Gen[String] = genRegexMatchingString(duration)
 Gen.listOfN(3, genDurationString).apply(Gen.Parameters.default, Seed(1046531L))
 ```
 
+Were all of those results input that you intended your regular expression to accept?
+
+This generation is done efficiently as opposed to generating a bunch of random values and then filtering the ones that don't match the regular expression (which would quickly lead to Scalacheck giving up on generating matching values).
+
 Sometimes you may want to generate both matches and non-matches for your regular expression to make sure that both cases are handled. There are various `Gen` instances that will generate input that matches the regular expresssion roughly half of the time.
 
 ```scala mdoc:silent
 val genDurationCandidateString: Gen[String] =
   Gen.resize(12, genRegexCandidateString(duration))
 
 val genExamples: Gen[List[(String, Option[Duration])]] =
-  Gen.listOfN(3, genDurationCandidateString).map(candidates =>
+  Gen.listOfN(4, genDurationCandidateString).map(candidates =>
     candidates.map(candidate => (candidate, durationParser.parseOnlyS(candidate)))
   )
 ```
 
 ```scala mdoc
-genExamples.apply(Gen.Parameters.default, Seed(1046531L))
+genExamples.apply(Gen.Parameters.default, Seed(1046533L))
 ```
 
 ### random regular expressions
@@ -154,13 +158,10 @@ regexesAndMatches.map(x =>
 Sometimes you may want to generate both matches and non-matches for your random regular expression to make sure that both cases are handled. There are various `Gen` instances for `RegexAndCandidate` that will generate random regular expressions along with data that matches the regular expresssion roughly half of the time.
 
 ```scala mdoc:silent
-val regexAndCandidateGen: Gen[RegexAndCandidate[Char, Double]] =
-  Gen.resize(12, genAlphaNumRegexAndCandidate)
-
 val regexesAndCandidatesGen: Gen[List[RegexAndCandidate[Char, Double]]] =
-  Gen.listOfN(4, regexAndCandidateGen)
+  Gen.listOfN(4, genAlphaNumRegexAndCandidate)
 
-val regexesAndCandidates: List[RegexAndCandidate[Char, Double]] = regexesAndCandidatesGen.apply(Gen.Parameters.default.withSize(30), Seed(105771L)).get
+val regexesAndCandidates: List[RegexAndCandidate[Char, Double]] = regexesAndCandidatesGen.apply(Gen.Parameters.default.withSize(15), Seed(105375L)).get
 ```
 
 ```scala mdoc

parser/src/main/scala/Parser.scala

@@ -19,18 +19,24 @@ object Parser {
   sealed abstract class RepeatCount extends Product with Serializable {
     def min: Int = this match {
       case RepeatCount.Exact(n) => n
-      case RepeatCount.Range(lower, _) => lower
+      case RepeatCount.Range(lower, _, _) => lower
     }
 
     def max: Option[Int] = this match {
       case RepeatCount.Exact(n) => Some(n)
-      case RepeatCount.Range(_, upper) => upper
+      case RepeatCount.Range(_, upper, _) => upper
+    }
+
+    def onRegex(regex: RegexC[Unit]): RegexC[Unit] = this match {
+      case RepeatCount.Exact(n) => regex.count(n).void
+      case RepeatCount.Range(min, max, g) => regex.repeat(min, max, g).void
     }
   }
 
   object RepeatCount {
     final case class Exact(n: Int) extends RepeatCount
-    final case class Range(lowerInclusive: Int, upperInclusive: Option[Int]) extends RepeatCount
+    final case class Range(lowerInclusive: Int, upperInclusive: Option[Int], greediness: Greediness)
+        extends RepeatCount
   }
 
   private val escapableCharToLit: Map[Char, Char] = specialNonCharClassCharToLit + ('-' -> '-')
@@ -115,9 +121,11 @@ object Parser {
   def repeatCount[_: P]: P[RepeatCount] =
     P(
       "{" ~/ (
-        (posInt ~ "," ~/ posInt.?).map { case (l, h) => RepeatCount.Range(l, h) } |
-          posInt.map(RepeatCount.Exact(_))
-      ) ~ "}").opaque("repeat count such as '{3}', '{1,4}', or '{3,}'")
+        (posInt ~ "," ~/ posInt.? ~/ "}" ~/ (P("?").map(_ => Greediness.NonGreedy) | Pass(
+          Greediness.Greedy))).map { case (l, h, g) => RepeatCount.Range(l, h, g) } |
+          (posInt.map(RepeatCount.Exact(_)) ~ "}")
+      )
+    ).opaque("repeat count such as '{3}', '{1,4}', `{1, 4}?`, '{3,}', or `{3,}?")
 
   def charOrRange[_: P]: P[Match.MatchSet[Char]] =
     matchCharRange.map(r => MatchSet.allow(Diet.fromRange(r))) |
@@ -187,12 +195,11 @@ object Parser {
 
   def factor[_: P]: P[RegexC[Unit]] = P {
     base.flatMap { r =>
-      // TODO greediness
-      // TODO voids
-      P("*").map(_ => r.chain(Greediness.Greedy).void) |
+      P("*?").map(_ => r.star(Greediness.NonGreedy).void) |
+        P("*").map(_ => r.star(Greediness.Greedy).void) |
         P("+").map(_ => r.oneOrMore(Greediness.Greedy).void) |
         P("?").map(_ => r.optional.void) |
-        repeatCount.map(count => r.repeat(count.min, count.max, Greediness.Greedy).void) |
+        repeatCount.map(_.onRegex(r)) |
         Pass(r)
     }
   }

regex-gen/src/main/scala/CharRegexGen.scala

@@ -63,5 +63,8 @@ object CharRegexGen {
     regexMatchingStringGenFromDiet(supportedCharacters)
 
   def genRegexCandidateString[Out]: RegexC[Out] => Gen[String] =
-    r => Gen.oneOf(regexMatchingStringGenFromDiet(supportedCharacters)(r), arbitrary[String])
+    r =>
+      Gen.oneOf(
+        regexMatchingStringGenFromDiet(supportedCharacters)(r),
+        Gen.frequency(2 -> Gen.asciiStr, 1 -> arbitrary[String]))
 }

regex-gen/src/main/scala/RegexGen.scala

@@ -42,7 +42,7 @@ object RegexGen {
 
   def genMatch[A: Discrete: Order](genA: Gen[A], genDietA: Gen[Diet[A]]): Gen[Match[A]] =
     Gen.frequency(
-      9 -> genA.map(Match.lit(_)),
+      16 -> genA.map(Match.lit(_)),
       3 -> genDietA.map(MatchSet.allow(_)),
       2 -> genDietA.map(MatchSet.forbid(_)),
       1 -> Gen.const(Match.wildcard)
@@ -82,7 +82,7 @@ object RegexGen {
     else
       Gen.frequency(
         // AndThen
-        5 -> (
+        20 -> (
           for {
             rIDepth <- Gen.choose(1, depth - 1)
             rIGen <- genRegexWithEv[In](cfg).apply(rIDepth)
@@ -95,7 +95,7 @@ object RegexGen {
           } yield combinator.andThen(rf, rI)
         ),
         // FMap
-        2 -> (for {
+        3 -> (for {
           regexGen <- genRegexWithEv[In](cfg).apply(depth - 1)
           r <- regexGen.evidence.regexGen
           f <- {
@@ -111,8 +111,24 @@ object RegexGen {
             nel <- depths.traverse(depth => genRegexWithDepth[In, Out](cfg, depth))
           } yield Regex.Or[In, Match[In], Out](nel)
         ),
+        // Repeat
+        3 -> (
+          for {
+            maxCount <- Gen.choose(1, math.min(depth - 1, 5))
+            rIGen <- genRegexWithEv[In](cfg).apply(depth - maxCount)
+            rI <- rIGen.evidence.regexGen
+            min <- Gen.chooseNum(0, maxCount)
+            max <- Gen.frequency(1 -> None, 5 -> Some(maxCount))
+            g <- arbitrary[Greediness]
+            z <- arbitrary[Out]
+            fold <- {
+              implicit val iCogen = rIGen.evidence.cogenOut
+              arbitrary[(Out, rIGen.T) => Out]
+            }
+          } yield combinator.repeatFold(rI, min, max, g, z)(fold)
+        ),
         // Star
-        1 -> (
+        2 -> (
           for {
             rIGen <- genRegexWithEv[In](cfg).apply(depth - 1)
             rI <- rIGen.evidence.regexGen
@@ -122,7 +138,7 @@ object RegexGen {
               implicit val iCogen = rIGen.evidence.cogenOut
               arbitrary[(Out, rIGen.T) => Out]
             }
-          } yield combinator.star(rI, g, z)(fold)
+          } yield combinator.starFold(rI, g, z)(fold)
         )
       )
   }

regex-gen/src/main/scala/RegexMatchGen.scala

@@ -42,6 +42,13 @@ object RegexMatchGen {
           star =
             λ[λ[i => (RegexM[In, i], Greediness, Out, (Out, i) => Out)] ~> λ[a => Gen[Stream[In]]]](
               t => Gen.containerOf[Stream, Stream[In]](outer.apply(t._1)).map(_.flatten)),
+          repeat =
+            λ[λ[i => (RegexM[In, i], Int, Option[Int], Greediness, Out, (Out, i) => Out)] ~> λ[
+              a => Gen[Stream[In]]]](t =>
+              for {
+                count <- Gen.chooseNum(t._2, math.max(t._2, t._3.getOrElse(5)))
+                nestedStream <- Gen.containerOfN[Stream, Stream[In]](count, outer.apply(t._1))
+              } yield nestedStream.flatten),
           mapped =
             λ[λ[a => (RegexM[In, a], a => Out)] ~> λ[a => Gen[Stream[In]]]](t => outer.apply(t._1)),
           or = alternatives => Gen.oneOf(alternatives.toList).flatMap(apply),

regex/src/main/scala/Regex.scala

@@ -25,8 +25,8 @@ import cats.implicits._
  * and [[ceedubs.irrec.regex.RegexCOps]].
  */
 sealed abstract class Regex[-In, +M, Out] extends Serializable {
-  def chain(greediness: Greediness): Regex[In, M, Chain[Out]] =
-    combinator.chain(this, greediness)
+  def star(greediness: Greediness): Regex[In, M, Chain[Out]] =
+    combinator.star(this, greediness)
 
   def many: Regex[In, M, Chain[Out]] = combinator.many(this)
 
@@ -85,6 +85,38 @@ object Regex {
       extends Regex[In, M, Out] {
     type Init = I
   }
+
+  final case class Repeat[-In, +M, I, Out](
+    r: Regex[In, M, I],
+    minInclusive: Int,
+    maxInclusive: Option[Int],
+    greediness: Greediness,
+    z: Out,
+    fold: (Out, I) => Out)
+      extends Regex[In, M, Out] {
+    type Init = I
+
+    def expand: Regex[In, M, Out] = {
+      val tail = maxInclusive.fold(combinator.star(r, greediness).some) { max =>
+        if (max <= minInclusive) None
+        else {
+          (0 to (max - minInclusive)).toList.toNel.map { counts =>
+            val orderedCounts = greediness match {
+              // TODO reversed?
+              case Greediness.Greedy => counts.reverse
+              case Greediness.NonGreedy => counts
+            }
+            Regex.Or(orderedCounts.map(i => expandedCount(i, r)))
+          }
+        }
+      }
+      val head = expandedCount(minInclusive, r)
+      tail
+        .fold(head)(tail => head.map2(tail)(_ concat _))
+        .map(_.foldLeft(z)(fold))
+    }
+  }
+
   // TODO efficiently handle with NFA
   final case class Void[-In, +M, I](r: Regex[In, M, I]) extends Regex[In, M, Unit] {
     type Init = I
@@ -97,6 +129,7 @@ object Regex {
     case Eps => F.pure(Eps)
     case x @ Fail() => F.pure(x)
     case Star(r, g, z, fold) => traverseM(r)(f).map(Star(_, g, z, fold))
+    case Repeat(r, min, max, g, z, fold) => traverseM(r)(f).map(Repeat(_, min, max, g, z, fold))
     case FMap(r, g) => traverseM(r)(f).map(FMap(_, g))
     case Or(alternatives) => alternatives.traverse(traverseM(_)(f)).map(Or(_))
     case AndThen(l, r) => traverseM(l)(f).map2(traverseM(r)(f))(AndThen(_, _))
@@ -109,6 +142,8 @@ object Regex {
     elem: (M, In => Option[Out]) => R,
     andThen: λ[i => (Regex[In, M, i => Out], Regex[In, M, i])] ~> λ[a => R],
     star: λ[i => (Regex[In, M, i], Greediness, Out, (Out, i) => Out)] ~> λ[a => R],
+    repeat: λ[i => (Regex[In, M, i], Int, Option[Int], Greediness, Out, (Out, i) => Out)] ~> λ[
+      a => R],
     mapped: λ[a => (Regex[In, M, a], a => Out)] ~> λ[a => R],
     or: NonEmptyList[Regex[In, M, Out]] => R,
     void: Is[Unit, Out] => Regex[In, M, ?] ~> λ[a => R]
@@ -117,6 +152,7 @@ object Regex {
     case Or(alternatives) => or(alternatives)
     case e: Elem[In, M, Out] => elem(e.metadata, e.apply)
     case Star(r, g, z, f) => star((r, g, z, f))
+    case Repeat(r, min, max, g, z, f) => repeat((r, min, max, g, z, f))
     case FMap(r, f) => mapped((r, f))
     case Eps => eps(Is.refl[Unit])
     case Fail() => fail()
@@ -152,7 +188,7 @@ object Regex {
   // TODO Stream is deprecated in 2.13, right?
   // TODO use Cont/ContT?
   // TODO return a custom type?
-  def compileCont[In, M, A, R](
+  private def compileCont[In, M, A, R](
     re: Regex[In, (ThreadId, M), A]): Cont[A => Stream[Thread[In, R]]] => Stream[Thread[In, R]] = {
     type ContOut = Cont[A => Stream[Thread[In, R]]] => Stream[Thread[In, R]]
     Regex.fold[In, (ThreadId, M), A, ContOut](
@@ -200,6 +236,18 @@ object Regex {
             threads(z, _)
           }
         },
+      repeat = new (λ[i => (
+        Regex[In, (ThreadId, M), i],
+        Int,
+        Option[Int],
+        Greediness,
+        A,
+        (A, i) => A)] ~> λ[a => ContOut]) {
+        def apply[i](
+          t: (Regex[In, (ThreadId, M), i], Int, Option[Int], Greediness, A, (A, i) => A)): ContOut =
+          sys.error(
+            "compileCont called with a Repeat instance that hadn't been expanded. This should never happen.")
+      },
       mapped = new (λ[a => (Regex[In, (ThreadId, M), a], a => A)] ~> λ[a => ContOut]) {
         def apply[i](t: (Regex[In, (ThreadId, M), i], i => A)): ContOut = {
           val rc = compileCont[In, M, i, R](t._1)
@@ -216,14 +264,42 @@ object Regex {
     )(re)
   }
 
+  private def expandRepeat[In, M]: Regex[In, M, ?] ~> Regex[In, M, ?] =
+    new (Regex[In, M, ?] ~> Regex[In, M, ?]) {
+      def apply[A](r: Regex[In, M, A]): Regex[In, M, A] = r match {
+        case r: Repeat[_, _, _, _] => r.expand
+        case r => r
+      }
+    }
+
   def compile[In, M, Out](r: Regex[In, M, Out]): ParseState[In, Out] = {
     val threads =
       Regex
-        .compileCont(assignThreadIds(r))
+        .compileCont(assignThreadIds(transformRecursive(expandRepeat)(r)))
         .apply(Cont.Single((out: Out) => Stream(Thread.Accept[In, Out](out))))
     ParseState.fromThreads(threads)
   }
 
+  def transformRecursive[In, M](
+    f: Regex[In, M, ?] ~> Regex[In, M, ?]): Regex[In, M, ?] ~> Regex[In, M, ?] =
+    new (Regex[In, M, ?] ~> Regex[In, M, ?]) {
+      def apply[A](fa: Regex[In, M, A]): Regex[In, M, A] = fa match {
+        case Eps => f(Eps)
+        case x @ Fail() => f(x)
+        case x: Elem[_, _, _] => f(x)
+        case AndThen(l, r) => f(AndThen(apply(l), apply(r)))
+        case Or(alternatives) => f(Or(alternatives.map(apply(_))))
+        case FMap(r, g) => f(FMap(apply(r), g))
+        case Star(r, greediness, z, fold) => f(Star(apply(r), greediness, z, fold))
+        case Repeat(r, minInclusive, maxInclusive, greediness, z, fold) =>
+          f(Repeat(apply(r), minInclusive, maxInclusive, greediness, z, fold))
+        case Void(r) => f(Void(apply(r)))
+      }
+    }
+
+  private def expandedCount[In, M, Out](n: Int, r: Regex[In, M, Out]): Regex[In, M, Chain[Out]] =
+    Chain.fromSeq(1 to n).traverse(_ => r)
+
   // TODO optimize
   // TODO naming/documentation
   def matcher[F[_]: Foldable, In, M, Out](r: Regex[In, M, Out]): F[In] => Boolean = {

regex/src/main/scala/RegexOps.scala

@@ -11,7 +11,8 @@ final class RegexOps[In, M, Out](private val r: Regex[In, M, Out]) extends AnyVa
   def either[Out2](o: Regex[In, M, Out2]): Regex[In, M, Either[Out, Out2]] =
     combinator.either(r, o)
 
-  def star[Out2](g: Greediness, z: Out2)(fold: (Out2, Out) => Out2) = combinator.star(r, g, z)(fold)
+  def starFold[Out2](g: Greediness, z: Out2)(fold: (Out2, Out) => Out2) =
+    combinator.starFold(r, g, z)(fold)
 
   def matcher[F[_]: Foldable]: F[In] => Boolean = Regex.matcher(r)