fix #30335, regression in intersection of unions of typevars

src/subtype.c

@@ -1333,12 +1333,17 @@ static jl_value_t *intersect_all(jl_value_t *x, jl_value_t *y, jl_stenv_t *e);
 // intersect in nested union environment, similar to subtype_ccheck
 static jl_value_t *intersect_aside(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int depth)
 {
- jl_value_t *res;
+ // band-aid for #30335
+ if (x == (jl_value_t*)jl_any_type && !jl_is_typevar(y))
+ return y;
+ if (y == (jl_value_t*)jl_any_type && !jl_is_typevar(x))
+ return x;
+
  int savedepth = e->invdepth;
  jl_unionstate_t oldRunions = e->Runions;
  e->invdepth = depth;
 
- res = intersect_all(x, y, e);
+ jl_value_t *res = intersect_all(x, y, e);
 
  e->Runions = oldRunions;
  e->invdepth = savedepth;
@@ -1466,6 +1471,8 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int
  return (jl_value_t*)b;
  }
  else if (bb->constraintkind == 2) {
+ // TODO: removing this case fixes many test_brokens in test/subtype.jl
+ // but breaks other tests.
  if (!subtype_in_env(a, bb->ub, e))
  return jl_bottom_type;
  jl_value_t *lb = simple_join(bb->lb, a);
@@ -1624,6 +1631,10 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
  // you can construct `T{x} where x` even if T's parameter is actually
  // limited. in that case we might get an invalid instantiation here.
  res = jl_substitute_var(res, vb->var, varval);
+ // simplify chains of UnionAlls where bounds become equal
+ while (jl_is_unionall(res) && obviously_egal(((jl_unionall_t*)res)->var->lb,
+ ((jl_unionall_t*)res)->var->ub))
+ res = jl_instantiate_unionall((jl_unionall_t*)res, ((jl_unionall_t*)res)->var->lb);
  }
  JL_CATCH {
  res = jl_bottom_type;

test/subtype.jl

@@ -1399,3 +1399,24 @@ end
 @testintersect(Tuple{Pair{Int64,2}, NTuple},
  Tuple{Pair{F,N},Tuple{Vararg{F,N}}} where N where F,
  Tuple{Pair{Int64,2}, Tuple{Int64,Int64}})
+
+# issue #30335
+@testintersect(Tuple{Any,Rational{Int},Int},
+ Tuple{LT,R,I} where LT<:Union{I, R} where R<:Rational{I} where I<:Integer,
+ Tuple{LT,Rational{Int},Int} where LT<:Union{Rational{Int},Int})
+
+#@testintersect(Tuple{Any,Tuple{Int},Int},
+# Tuple{LT,R,I} where LT<:Union{I, R} where R<:Tuple{I} where I<:Integer,
+# Tuple{LT,Tuple{Int},Int} where LT<:Union{Tuple{Int},Int})
+# fails due to this:
+let U = Tuple{Union{LT, LT1},Union{R, R1},Int64} where LT1<:R1 where R1<:Tuple{Int64} where LT<:Int64 where R<:Tuple{Int64},
+ U2 = Union{Tuple{LT,R,Int64} where LT<:Int64 where R<:Tuple{Int64}, Tuple{LT,R,Int64} where LT<:R where R<:Tuple{Int64}},
+ V = Tuple{Union{Tuple{Int},Int},Tuple{Int},Int},
+ V2 = Tuple{L,Tuple{Int},Int} where L<:Union{Tuple{Int},Int}
+ @test U == U2
+ @test U == V
+ @test U == V2
+ @test V == V2
+ @test U2 == V
+ @test_broken U2 == V2
+end