Avoid bidirectional GADT typebounds from fullBounds

Author: dwijnand

compiler/src/dotty/tools/dotc/core/GadtConstraint.scala

@@ -71,12 +71,26 @@ class GadtConstraint private (
     externalize(constraint.nonParamBounds(param)).bounds
 
   def fullLowerBound(param: TypeParamRef)(using Context): Type =
-    constraint.minLower(param).foldLeft(nonParamBounds(param).lo) {
+    val self = externalize(param)
+    constraint.minLower(param).filterNot { p =>
+      val sym = paramSymbol(p)
+      sym.name.is(NameKinds.UniqueName) && {
+        val hi = sym.info.hiBound
+        !hi.isExactlyAny && self <:< hi
+      }
+    }.foldLeft(nonParamBounds(param).lo) {
       (t, u) => t | externalize(u)
     }
 
   def fullUpperBound(param: TypeParamRef)(using Context): Type =
-    constraint.minUpper(param).foldLeft(nonParamBounds(param).hi) { (t, u) =>
+    val self = externalize(param)
+    constraint.minUpper(param).filterNot { p =>
+      val sym = paramSymbol(p)
+      sym.name.is(NameKinds.UniqueName) && {
+        val lo = sym.info.loBound
+        !lo.isExactlyNothing && lo <:< self
+      }
+    }.foldLeft(nonParamBounds(param).hi) { (t, u) =>
       val eu = externalize(u)
       // Any as the upper bound means "no bound", but if F is higher-kinded,
       // Any & F = F[_]; this is wrong for us so we need to short-circuit
@@ -96,6 +110,10 @@ class GadtConstraint private (
   def tvarOrError(sym: Symbol)(using Context): TypeVar =
     mapping(sym).ensuring(_ != null, i"not a constrainable symbol: $sym").uncheckedNN
 
+  private def paramSymbol(p: TypeParamRef): Symbol = reverseMapping(p) match
+    case sym: Symbol => sym
+    case null        => NoSymbol
+
   @tailrec final def stripInternalTypeVar(tp: Type): Type = tp match
     case tv: TypeVar =>
       val inst = constraint.instType(tv)

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1765,7 +1765,7 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
           else report.error(new DuplicateBind(b, cdef), b.srcPos)
           if (!ctx.isAfterTyper) {
             val bounds = ctx.gadt.fullBounds(sym)
-            if (bounds != null) sym.info = bounds
+            if (bounds != null) sym.info = checkNonCyclic(sym, bounds, reportErrors = true)
           }
           b
         case t: UnApply if t.symbol.is(Inline) => Inlines.inlinedUnapply(t)

tests/pos/i14287.min.scala

@@ -0,0 +1,7 @@
+enum Foo[+F[_]]:
+  case Bar[B[_]](value: Foo[B]) extends Foo[B]
+
+class Test:
+  def test[X[_]](foo: Foo[X]): Foo[X] = foo match
+    case Foo.Bar(Foo.Bar(x)) => Foo.Bar(x)
+    case _                   => foo

tests/pos/i14287.scala

@@ -0,0 +1,16 @@
+// scalac: -Yno-deep-subtypes
+enum Free[+F[_], A]:
+  case Return(a: A)
+  case Suspend(s: F[A])
+  case FlatMap[F[_], A, B](
+    s: Free[F, A],
+    f: A => Free[F, B]) extends Free[F, B]
+
+  def flatMap[F2[x] >: F[x], B](f: A => Free[F2,B]): Free[F2,B] =
+    FlatMap(this, f)
+
+  @scala.annotation.tailrec
+  final def step: Free[F, A] = this match
+    case FlatMap(FlatMap(fx, f), g) => fx.flatMap(x => f(x).flatMap(y => g(y))).step
+    case FlatMap(Return(x), f) => f(x).step
+    case _ => this

tests/pos/i15523.avoid.scala

@@ -0,0 +1,8 @@
+// scalac: -Werror
+// like the original, but with a case body `a`
+// which caused type avoidance to infinitely recurse
+sealed trait Parent
+final case class Leaf[A, B >: A](a: A, b: B) extends Parent
+
+def run(x: Parent) = x match
+  case Leaf(a, _) => a

tests/pos/i15523.scala

@@ -0,0 +1,7 @@
+// scalac: -Werror
+sealed trait Parent
+final case class Leaf[A, B >: A](a: A, b: B) extends Parent
+
+def run(x: Parent): Unit = x match {
+  case Leaf(a, b) =>
+}

tests/pos/i16777.scala

@@ -0,0 +1,52 @@
+// scalac: -Ykind-projector:underscores
+
+sealed abstract class Free[+S[_, _], +E, +A] {
+  @inline final def flatMap[S1[e, a] >: S[e, a], B, E1 >: E](fun: A => Free[S1, E1, B]): Free[S1, E1, B] = Free.FlatMapped[S1, E, E1, A, B](this, fun)
+  @inline final def map[B](fun: A => B): Free[S, E, B] = flatMap(a => Free.pure[S, B](fun(a)))
+  @inline final def as[B](as: => B): Free[S, E, B] = map(_ => as)
+  @inline final def *>[S1[e, a] >: S[e, a], B, E1 >: E](sc: Free[S1, E1, B]): Free[S1, E1, B] = flatMap(_ => sc)
+  @inline final def <*[S1[e, a] >: S[e, a], B, E1 >: E](sc: Free[S1, E1, B]): Free[S1, E1, A] = flatMap(r => sc.as(r))
+
+  @inline final def void: Free[S, E, Unit] = map(_ => ())
+
+  // FIXME: Scala 3.1.4 bug: false unexhaustive match warning
+  /// @nowarn("msg=pattern case: Free.FlatMapped")
+  @inline final def foldMap[S1[e, a] >: S[e, a], G[+_, +_]](transform: S1 ~>> G)(implicit G: Monad2[G]): G[E, A] = {
+    this match {
+      case Free.Pure(a) => G.pure(a)
+      case Free.Suspend(a) => transform.apply(a)
+      case Free.FlatMapped(sub, cont) =>
+        sub match {
+          case Free.FlatMapped(sub2, cont2) => sub2.flatMap(a => cont2(a).flatMap(cont)).foldMap(transform)
+          case another => G.flatMap(another.foldMap(transform))(cont(_).foldMap(transform))
+        }
+    }
+  }
+}
+
+trait ~>>[-F[_, _], +G[_, _]] {
+  def apply[E, A](f: F[E, A]): G[E, A]
+}
+
+object Free {
+  @inline def pure[S[_, _], A](a: A): Free[S, Nothing, A] = Pure(a)
+  @inline def lift[S[_, _], E, A](s: S[E, A]): Free[S, E, A] = Suspend(s)
+
+  final case class Pure[S[_, _], A](a: A) extends Free[S, Nothing, A] {
+    override def toString: String = s"Pure:[$a]"
+  }
+  final case class Suspend[S[_, _], E, A](a: S[E, A]) extends Free[S, E, A] {
+    override def toString: String = s"Suspend:[$a]"
+  }
+  final case class FlatMapped[S[_, _], E, E1 >: E, A, B](sub: Free[S, E, A], cont: A => Free[S, E1, B]) extends Free[S, E1, B] {
+    override def toString: String = s"FlatMapped:[sub=$sub]"
+  }
+}
+
+type Monad2[F[+_, +_]] = Monad3[λ[(`-R`, `+E`, `+A`) => F[E, A]]]
+
+trait Monad3[F[-_, +_, +_]] {
+  def flatMap[R, E, A, B](r: F[R, E, A])(f: A => F[R, E, B]): F[R, E, B]
+  def flatten[R, E, A](r: F[R, E, F[R, E, A]]): F[R, E, A] = flatMap(r)(identity)
+  def pure[A](a: A): F[Any, Nothing, A]
+}