inference: limit single-level nested Type signature (JuliaLang#40379)

Previously `type_more_complex` returns `false` for `Type{Type{...}}`
compared against `Type{...}`. Per comment there, this should return `true`,
but the code was ordered incorrect (it was also missing support for
Vararg, after the recent changes).

Fixes JuliaLang#40336

Co-authored-by: Shuhei Kadowaki <aviatesk@gmail.com>

base/compiler/typelimits.jl

@@ -128,7 +128,15 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
         end
         return Vararg{VaT}
     elseif isa(t, DataType)
-        if isa(c, DataType)
+        if isa(c, Core.TypeofVararg)
+            # Tuple{Vararg{T}} --> Tuple{T} is OK
+            return _limit_type_size(t, c.T, sources, depth, 0)
+        elseif isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
+            tt = unwrap_unionall(t.parameters[1])
+            (!isa(tt, DataType) || isType(tt)) && (depth += 1)
+            is_derived_type_from_any(tt, sources, depth) && return t
+            return Type
+        elseif isa(c, DataType)
             tP = t.parameters
             cP = c.parameters
             if t.name === c.name && !isempty(cP)
@@ -158,15 +166,6 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
                     return Tuple{Q...}
                 end
             end
-        elseif isa(c, Core.TypeofVararg)
-            # Tuple{Vararg{T}} --> Tuple{T} is OK
-            return _limit_type_size(t, c.T, sources, depth, 0)
-        end
-        if isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
-            tt = unwrap_unionall(t.parameters[1])
-            if isa(tt, DataType) && !isType(tt)
-                is_derived_type_from_any(tt, sources, depth) && return t
-            end
         end
         if allowed_tuplelen < 1 && t.name === Tuple.name
             return Any
@@ -226,17 +225,27 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe
         return t !== 1 && !(0 <= t < c) # alternatively, could use !(abs(t) <= abs(c) || abs(t) < n) for some n
     end
     # base case for data types
-    if isa(t, DataType)
+    if isa(t, Core.TypeofVararg)
+        if isa(c, Core.TypeofVararg)
+            return type_more_complex(unwrapva(t), unwrapva(c), sources, depth + 1, tupledepth, 0)
+        end
+    elseif isa(t, DataType)
         tP = t.parameters
-        if isa(c, DataType) && t.name === c.name
+        if isa(c, Core.TypeofVararg)
+            return type_more_complex(t, unwrapva(c), sources, depth, tupledepth, 0)
+        elseif isType(t) # allow taking typeof any source type anywhere as Type{...}, as long as it isn't nesting Type{Type{...}}
+            tt = unwrap_unionall(t.parameters[1])
+            (!isa(tt, DataType) || isType(tt)) && (depth += 1)
+            return !is_derived_type_from_any(tt, sources, depth)
+        elseif isa(c, DataType) && t.name === c.name
             cP = c.parameters
             length(cP) < length(tP) && return true
             length(cP) > length(tP) && !isvarargtype(tP[end]) && depth == 1 && return false
             ntail = length(cP) - length(tP) # assume parameters were dropped from the tuple head
             # allow creating variation within a nested tuple, but only so deep
             if t.name === Tuple.name && tupledepth > 0
                 tupledepth -= 1
-            elseif !isvarargtype(t)
+            else
                 tupledepth = 0
             end
             isgenerator = (t.name.name === :Generator && t.name.module === _topmod(t.name.module))
@@ -258,15 +267,6 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe
                 type_more_complex(tPi, cPi, sources, depth + 1, tupledepth, 0) && return true
             end
             return false
-        elseif isvarargtype(c)
-            return type_more_complex(t, unwrapva(c), sources, depth, tupledepth, 0)
-        end
-        if isType(t) # allow taking typeof any source type anywhere as Type{...}, as long as it isn't nesting Type{Type{...}}
-            tt = unwrap_unionall(t.parameters[1])
-            if isa(tt, DataType) && !isType(tt)
-                is_derived_type_from_any(tt, sources, depth) || return true
-                return false
-            end
         end
     end
     return true

test/compiler/inference.jl

@@ -44,6 +44,13 @@ let t = Tuple{Ref{T},T,T} where T, c = Tuple{Ref, T, T} where T # #36407
     @test t <: Core.Compiler.limit_type_size(t, c, Union{}, 1, 100)
 end
 
+let # 40336
+    t = Type{Type{Int}}
+    c = Type{Int}
+    r = Core.Compiler.limit_type_size(t, c, c, 100, 100)
+    @test t !== r && t <: r
+end
+
 @test Core.Compiler.unionlen(Union{}) == 1
 @test Core.Compiler.unionlen(Int8) == 1
 @test Core.Compiler.unionlen(Union{Int8, Int16}) == 2
@@ -3171,8 +3178,6 @@ end
 end
 
 @testset "constant prop' for union split signature" begin
-    anonymous_module() = Core.eval(@__MODULE__, :(module $(gensym()) end))::Module
-
     # indexing into tuples really relies on constant prop', and we will get looser result
     # (`Union{Int,String,Char}`) if constant prop' doesn't happen for splitunion signatures
     tt = (Union{Tuple{Int,String},Tuple{Int,Char}},)
@@ -3191,7 +3196,7 @@ end
         b
     end == Any[Union{String,Char}]
 
-    @test (@eval anonymous_module() begin
+    @test (@eval Module() begin
         struct F32
             val::Float32
             _v::Int
@@ -3205,7 +3210,7 @@ end
         end
     end) == Any[Union{Float32,Float64}]
 
-    @test (@eval anonymous_module() begin
+    @test (@eval Module() begin
         struct F32
             val::Float32
             _v
@@ -3243,3 +3248,25 @@ end
         Some(0x2)
     end
 end == [Union{Some{Float64}, Some{Int}, Some{UInt8}}]
+
+# https://github.com/JuliaLang/julia/issues/40336
+@testset "make sure a call with signatures with recursively nested Types terminates" begin
+    @test @eval Module() begin
+        f(@nospecialize(t)) = f(Type{t})
+
+        code_typed() do
+            f(Int)
+        end
+        true
+    end
+
+    @test @eval Module() begin
+        f(@nospecialize(t)) = tdepth(t) == 10 ? t : f(Type{t})
+        tdepth(@nospecialize(t)) = isempty(t.parameters) ? 1 : 1+tdepth(t.parameters[1])
+
+        code_typed() do
+            f(Int)
+        end
+        true
+    end
+end