From f9a8d4538aaaeb6a155880e7e7f94499abd0b220 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Wed, 9 Dec 2020 19:01:44 -0500
Subject: [PATCH] fix #38423, another stack overflow in method definition

also fixes #36544 and fixes #36804
---
 src/subtype.c   | 15 ++++++++++++++-
 test/subtype.jl | 20 ++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/subtype.c b/src/subtype.c
index e3cd777a42c29c..8b8e8d187e81af 100644
--- a/src/subtype.c
+++ b/src/subtype.c
@@ -2892,6 +2892,19 @@ static int compareto_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e, int cmp)
     return ans;
 }
 
+// See if var y is reachable from x via bounds; used to avoid cycles.
+static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
+{
+    if (x == (jl_value_t*)y)
+        return 1;
+    if (!jl_is_typevar(x))
+        return 0;
+    jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
+    if (xv == NULL)
+        return 0;
+    return reachable_var(xv->ub, y, e) || reachable_var(xv->lb, y, e);
+}
+
 // `param` means we are currently looking at a parameter of a type constructor
 // (as opposed to being outside any type constructor, or comparing variable bounds).
 // this is used to record the positions where type variables occur for the
@@ -2970,7 +2983,7 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa
                     assert(yy->ub != y);
                     assert(yy->lb != y);
                 }
-                if (xx) {
+                if (xx && !reachable_var(y, (jl_tvar_t*)x, e)) {
                     if (!compareto_var(y, (jl_tvar_t*)x, e, -1))
                         xx->lb = y;
                     if (!compareto_var(y, (jl_tvar_t*)x, e,  1))
diff --git a/test/subtype.jl b/test/subtype.jl
index 92e9d0c75f35c8..abeced2a7988cc 100644
--- a/test/subtype.jl
+++ b/test/subtype.jl
@@ -1814,3 +1814,23 @@ end
 
 # issue #24333
 @test_broken (Type{Union{Ref,Cvoid}} <: Type{Union{T,Cvoid}} where T)
+
+# issue #38423
+let
+    Either{L, R} = Union{Ref{L}, Val{R}}
+    A = Tuple{Type{Ref{L}}, Type{Either{L, <:Any}}} where L
+    B = Tuple{Type{Ref{L2}}, Type{Either{L1, R}}} where {L1, R, L2 <: L1}
+    I = typeintersect(A, B)
+    @test I != Union{}
+    @test_broken I <: A
+    @test_broken I <: B
+end
+
+# issue #36544
+let A = Tuple{T, Ref{T}, T} where {T},
+    B = Tuple{T, T, Ref{T}} where {T}
+    I = typeintersect(A, B)
+    @test I != Union{}
+    @test_broken I <: A
+    @test_broken I <: B
+end