From 53ba3c8c1031d2ba47b2a089c95f4c27244ac57b Mon Sep 17 00:00:00 2001
From: Dale Wijnand <dale.wijnand@gmail.com>
Date: Sat, 17 Aug 2024 11:59:47 +0100
Subject: [PATCH] Force LazyRef in AvoidWildcardsMap

This is to avoid unneeded new LazyRef's.  LazyRefs, which are created
for recursive types, aren't cacheable, so if you TypeMap an AppliedType
with one, it will create a brand new instance.  OrderingConstraint#init
runs AvoidWildcardsMap on param bounds.  So when instDirection compares
the constraint bounds and the original param bounds (to calculate the
instantiate direction), because they are new instances they won't
shortcircuit, leading to a recursion overflow.  By forcing, it will eq
check and return true.

In particular, with i5877, which is a neg test of recursive overflows,
the new implementation of interpolateTypeVars does more type comparisons
(to calculate the instDirection).  If the LazyRefs are kept eq, then the
AppliedTypes in the bounds are also eq, and so the subtyping check
shortcircuits true, rather than overflowing.
---
 compiler/src/dotty/tools/dotc/core/Types.scala |  3 +++
 tests/neg-deep-subtype/i5877.scala             | 10 +++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/compiler/src/dotty/tools/dotc/core/Types.scala b/compiler/src/dotty/tools/dotc/core/Types.scala
index 8181b3c83acf..1195126c66af 100644
--- a/compiler/src/dotty/tools/dotc/core/Types.scala
+++ b/compiler/src/dotty/tools/dotc/core/Types.scala
@@ -6631,6 +6631,9 @@ object Types extends TypeUtils {
       range(atVariance(-variance)(apply(bounds.lo)), apply(bounds.hi))
     def apply(t: Type): Type = t match
       case t: WildcardType => mapWild(t)
+      case tp: LazyRef => mapOver(tp) match
+        case tp1: LazyRef if tp.ref eq tp1.ref => tp
+        case tp1                               => tp1
       case _ => mapOver(t)
 
   // ----- TypeAccumulators ----------------------------------------------------
diff --git a/tests/neg-deep-subtype/i5877.scala b/tests/neg-deep-subtype/i5877.scala
index ac58764a48e5..c1d5ea4a31c1 100644
--- a/tests/neg-deep-subtype/i5877.scala
+++ b/tests/neg-deep-subtype/i5877.scala
@@ -1,4 +1,4 @@
-object Main { // error // error
+object Main {
     def main(a: Array[String]): Unit = {
       println("you may not run `testHasThisType` - just check that it compiles")
       // comment lines after "// this line of code makes" comments to make it compilable again
@@ -18,25 +18,25 @@ object Main { // error // error
 
     // ---- ---- ---- ----
 
-    def testHasThisType(): Unit = { // error // error
+    def testHasThisType(): Unit = {
       def testSelf[PThis <: HasThisType[_ <: PThis]](that: HasThisType[PThis]): Unit = {
         val thatSelf = that.self()
         // that.self().type <: that.This
         assert(implicitly[thatSelf.type <:< that.This] != null)
       }
       val that: HasThisType[_] = Foo() // null.asInstanceOf
-      testSelf(that) // error: recursion limit exceeded // error
+      testSelf(that) // error: recursion limit exceeded
     }
 
 
-    def testHasThisType2(): Unit = { // error // error
+    def testHasThisType2(): Unit = {
       def testSelf[PThis <: HasThisType[_ <: PThis]](that: PThis with HasThisType[PThis]): Unit = {
         // that.type <: that.This
         assert(implicitly[that.type <:< that.This] != null)
       }
       val that: HasThisType[_] = Foo() // null.asInstanceOf
       // this line of code makes Dotty compiler infinite recursion (stopped only by overflow) - comment it to make it compilable again
-      testSelf(that) // error: recursion limit exceeded // error
+      testSelf(that) // error: recursion limit exceeded
     }
 
     // ---- ---- ---- ----