From 3210ac82413e34f7ad0f13add80b05d2c1505d2c Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Mon, 14 Dec 2020 12:49:33 -0500
Subject: [PATCH] fix #38423, another stack overflow in method definition
 (#38810)

also fixes #36544 and fixes #36804

(cherry picked from commit 6ef077c8369e1d78d88dfe982250b224e73d8cae)
---
 src/subtype.c   | 48 ++++++++++++++++++++++++++++++++++++++----------
 test/subtype.jl | 31 +++++++++++++++++++++++++++++--
 2 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/src/subtype.c b/src/subtype.c
index 5a6c0a9f075cb..baf5a91b11899 100644
--- a/src/subtype.c
+++ b/src/subtype.c
@@ -2892,6 +2892,29 @@ static int compareto_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e, int cmp)
     return ans;
 }
 
+// Check whether the environment already asserts x <: y via recorded bounds.
+// This is used to avoid adding redundant constraints that lead to cycles.
+// Note this is a semi-predicate: 1 => is a subtype, 0 => unknown
+static int subtype_by_bounds(jl_value_t *x, jl_value_t *y, jl_stenv_t *e)
+{
+    if (!jl_is_typevar(x) || !jl_is_typevar(y))
+        return 0;
+    return compareto_var(x, (jl_tvar_t*)y, e, -1) || compareto_var(y, (jl_tvar_t*)x, e, 1);
+}
+
+// See if var y is reachable from x via bounds; used to avoid cycles.
+static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
+{
+    if (x == (jl_value_t*)y)
+        return 1;
+    if (!jl_is_typevar(x))
+        return 0;
+    jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
+    if (xv == NULL)
+        return 0;
+    return reachable_var(xv->ub, y, e) || reachable_var(xv->lb, y, e);
+}
+
 // `param` means we are currently looking at a parameter of a type constructor
 // (as opposed to being outside any type constructor, or comparing variable bounds).
 // this is used to record the positions where type variables occur for the
@@ -2945,9 +2968,16 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa
                         return xlb;
                     return jl_bottom_type;
                 }
-                if (R) flip_vars(e);
-                int ccheck = subtype_in_env(xlb, yub, e) && subtype_in_env(ylb, xub, e);
-                if (R) flip_vars(e);
+                int ccheck;
+                if (yub == xub ||
+                    (subtype_by_bounds(xlb, yub, e) && subtype_by_bounds(ylb, xub, e))) {
+                    ccheck = 1;
+                }
+                else {
+                    if (R) flip_vars(e);
+                    ccheck = subtype_in_env(xlb, yub, e) && subtype_in_env(ylb, xub, e);
+                    if (R) flip_vars(e);
+                }
                 if (!ccheck)
                     return jl_bottom_type;
                 if (var_occurs_inside(xub, (jl_tvar_t*)y, 0, 0) && var_occurs_inside(yub, (jl_tvar_t*)x, 0, 0)) {
@@ -2963,18 +2993,16 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa
                 else
                     lb = simple_join(xlb, ylb);
                 if (yy) {
-                    if (!compareto_var(lb, (jl_tvar_t*)y, e, -1))
+                    if (!subtype_by_bounds(lb, y, e))
                         yy->lb = lb;
-                    if (!compareto_var(ub, (jl_tvar_t*)y, e,  1))
+                    if (!subtype_by_bounds(y, ub, e))
                         yy->ub = ub;
                     assert(yy->ub != y);
                     assert(yy->lb != y);
                 }
-                if (xx) {
-                    if (!compareto_var(y, (jl_tvar_t*)x, e, -1))
-                        xx->lb = y;
-                    if (!compareto_var(y, (jl_tvar_t*)x, e,  1))
-                        xx->ub = y;
+                if (xx && !reachable_var(y, (jl_tvar_t*)x, e)) {
+                    xx->lb = y;
+                    xx->ub = y;
                     assert(xx->ub != x);
                 }
                 JL_GC_POP();
diff --git a/test/subtype.jl b/test/subtype.jl
index 9765767a379a2..47026df924d3b 100644
--- a/test/subtype.jl
+++ b/test/subtype.jl
@@ -1787,8 +1787,7 @@ let X1 = Tuple{AlmostLU, Vector{T}} where T,
     # doesn't stack overflow
     @test I<:X1 || I<:X2
     actual = Tuple{AlmostLU{S, X} where X<:Matrix{S}, Vector{S}} where S<:Union{Float32, Float64}
-    @test I >: actual
-    @test_broken I == actual
+    @test I == actual
 end
 
 let
@@ -1820,3 +1819,31 @@ let A = Tuple{Type{T} where T<:Ref, Ref, Union{T, Union{Ref{T}, T}} where T<:Ref
     @test I == typeintersect(A,B)
     @test I == Tuple{Type{T}, Ref{T}, Union{Ref{T}, T}} where T<:Ref
 end
+
+# issue #38423
+let
+    Either{L, R} = Union{Ref{L}, Val{R}}
+    A = Tuple{Type{Ref{L}}, Type{Either{L, <:Any}}} where L
+    B = Tuple{Type{Ref{L2}}, Type{Either{L1, R}}} where {L1, R, L2 <: L1}
+    I = typeintersect(A, B)
+    @test I != Union{}
+    @test_broken I <: A
+    @test_broken I <: B
+end
+
+# issue #36804
+let
+    Either{L, R} = Union{Some{L}, Ref{R}}
+    f(::Type{Either{L2, R}}, ::Type{Either{L1, R}}) where {L1, R, L2 <: L1} = Either{L1, R}
+    f(::Type{Either{L, R1}}, ::Type{Either{L, R2}}) where {L, R1, R2 <: R1} = Either{L, R1}
+    @test f(Either{Int,Real}, Either{Int,Float32}) == Either{Int,Real}
+end
+
+# issue #36544
+let A = Tuple{T, Ref{T}, T} where {T},
+    B = Tuple{T, T, Ref{T}} where {T}
+    I = typeintersect(A, B)
+    @test I != Union{}
+    @test_broken I <: A
+    @test_broken I <: B
+end