fix #40814, fix type inference regression introduced in #40379

Fixes the inference regression, while retaining the cases addressed by
<#40379>.

The same `Type`-name comparison pass (i.e. a branch of `isa(c, DataType) && t.name === c.name`)
can lead to more accurate result by recursive comparison than `isType`
check pass (i.e. a branch of `isType(t)`), and preferring the former
over the latter fixes the regression.

But just swapping the branch will lead to <#40336>,
and so this PR also implements additional check to make sure `type_more_complex`
still detects a single-level nesting correctly (especially, the `tt === c` parts).

base/compiler/typelimits.jl

@@ -131,15 +131,10 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
         if isa(c, Core.TypeofVararg)
             # Tuple{Vararg{T}} --> Tuple{T} is OK
             return _limit_type_size(t, c.T, sources, depth, 0)
-        elseif isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
-            tt = unwrap_unionall(t.parameters[1])
-            (!isa(tt, DataType) || isType(tt)) && (depth += 1)
-            is_derived_type_from_any(tt, sources, depth) && return t
-            return Type
-        elseif isa(c, DataType)
+        elseif isa(c, DataType) && t.name === c.name
             tP = t.parameters
             cP = c.parameters
-            if t.name === c.name && !isempty(cP)
+            if !isempty(cP)
                 if t.name === Tuple.name
                     # for covariant datatypes (Tuple),
                     # apply type-size limit element-wise
@@ -166,6 +161,11 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
                     return Tuple{Q...}
                 end
             end
+        elseif isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
+            tt = unwrap_unionall(t.parameters[1])
+            (!isa(tt, DataType) || isType(tt) || tt === c) && (depth += 1) # `tt === c` avoids single-level nesting, i.e. `t === Type{c}`
+            is_derived_type_from_any(tt, sources, depth) && return t
+            return Type
         end
         if allowed_tuplelen < 1 && t.name === Tuple.name
             return Any
@@ -233,10 +233,6 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe
         tP = t.parameters
         if isa(c, Core.TypeofVararg)
             return type_more_complex(t, unwrapva(c), sources, depth, tupledepth, 0)
-        elseif isType(t) # allow taking typeof any source type anywhere as Type{...}, as long as it isn't nesting Type{Type{...}}
-            tt = unwrap_unionall(t.parameters[1])
-            (!isa(tt, DataType) || isType(tt)) && (depth += 1)
-            return !is_derived_type_from_any(tt, sources, depth)
         elseif isa(c, DataType) && t.name === c.name
             cP = c.parameters
             length(cP) < length(tP) && return true
@@ -267,6 +263,10 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe
                 type_more_complex(tPi, cPi, sources, depth + 1, tupledepth, 0) && return true
             end
             return false
+        elseif isType(t) # allow taking typeof any source type anywhere as Type{...}, as long as it isn't nesting Type{Type{...}}
+            tt = unwrap_unionall(t.parameters[1])
+            (!isa(tt, DataType) || isType(tt) || tt === c) && (depth += 1) # `tt === c` avoids single-level nesting, i.e. `t === Type{c}`
+            return !is_derived_type_from_any(tt, sources, depth)
         end
     end
     return true

test/compiler/inference.jl

@@ -3289,6 +3289,9 @@ end == [Union{Some{Float64}, Some{Int}, Some{UInt8}}]
     end
 end
 
+# https://github.com/JuliaLang/julia/issues/40814
+@test Base.return_types(NTuple{3,Int}, (Vector{Int},)) == Any[NTuple{3,Int}]
+
 # Make sure that const prop doesn't fall into cycles that aren't problematic
 # in the type domain
 f_recurse(x) = x > 1000000 ? x : f_recurse(x+1)