SILOptimizer: Fix devirtualization of static method returning 'Self' in a generic class

lib/AST/Type.cpp

@@ -3336,17 +3336,22 @@ const DependentMemberType *TypeBase::findUnresolvedDependentMemberType() {
 
 Type TypeBase::getSuperclassForDecl(const ClassDecl *baseClass) {
   Type t(this);
+
+  if (!t->getAnyNominal()) {
+    if (auto archetype = t->getAs<ArchetypeType>()) {
+      t = archetype->getSuperclass();
+    } else if (auto dynamicSelfTy = t->getAs<DynamicSelfType>()) {
+      t = dynamicSelfTy->getSelfType();
+    } else if (auto compositionTy = t->getAs<ProtocolCompositionType>()) {
+      t = compositionTy->getExistentialLayout().superclass;
+    }
+  }
+
   while (t) {
     // If we have a class-constrained archetype or class-constrained
     // existential, get the underlying superclass constraint.
     auto *nominalDecl = t->getAnyNominal();
-    if (!nominalDecl) {
-      assert(t->is<ArchetypeType>() || t->isExistentialType() &&
-             "expected a class, archetype or existential");
-      t = t->getSuperclass();
-      assert(t && "archetype or existential is not class constrained");
-      continue;
-    }
+    assert(nominalDecl && "expected nominal type here");
     assert(isa<ClassDecl>(nominalDecl) && "expected a class here");
 
     if (nominalDecl == baseClass)
@@ -3363,7 +3368,9 @@ TypeBase::getContextSubstitutions(const DeclContext *dc,
   assert(dc->isTypeContext());
   Type baseTy(this);
 
-  assert(!baseTy->hasLValueType() && !baseTy->is<AnyMetatypeType>());
+  assert(!baseTy->hasLValueType() &&
+         !baseTy->is<AnyMetatypeType>() &&
+         !baseTy->is<ErrorType>());
 
   // The resulting set of substitutions. Always use this to ensure we
   // don't miss out on NRVO anywhere.

lib/Sema/ConstraintSystem.cpp

@@ -390,14 +390,12 @@ Type ConstraintSystem::openUnboundGenericType(UnboundGenericType *unbound,
                                               ConstraintLocatorBuilder locator,
                                               OpenedTypeMap &replacements) {
   auto unboundDecl = unbound->getDecl();
-  if (unboundDecl->isInvalid())
-    return ErrorType::get(getASTContext());
 
   // If the unbound decl hasn't been validated yet, we have a circular
   // dependency that isn't being diagnosed properly.
   if (!unboundDecl->getGenericSignature()) {
     TC.diagnose(unboundDecl, diag::circular_reference);
-    return ErrorType::get(unbound);
+    return Type();
   }
 
   auto parentTy = unbound->getParent();
@@ -458,14 +456,19 @@ Type ConstraintSystem::openUnboundGenericType(
   if (!type->hasUnboundGenericType())
     return type;
 
-  return type.transform([&](Type type) -> Type {
+  type = type.transform([&](Type type) -> Type {
       if (auto unbound = type->getAs<UnboundGenericType>()) {
         OpenedTypeMap replacements;
         return openUnboundGenericType(unbound, locator, replacements);
       }
 
       return type;
     });
+
+  if (!type)
+    return ErrorType::get(getASTContext());
+
+  return type;
 }
 
 Type ConstraintSystem::openType(Type type, OpenedTypeMap &replacements) {

test/SILOptimizer/devirt_static_covariant_return.swift

@@ -0,0 +1,20 @@
+// RUN: %target-swift-frontend -O -whole-module-optimization %s -emit-sil -sil-verify-all | %FileCheck %s
+
+// CHECK-NOT: class_method
+
+// Static method returning Self in generic class
+public class Factory<T> {
+  public required init() {}
+
+  @inline(never) class func bar() -> Self {
+    return self.init()
+  }
+
+  @inline(never) class func foo() -> Self {
+    return bar()
+  }
+}
+
+public func foo(m: Factory<Int>.Type) {
+  m.foo()
+}

validation-test/compiler_crashers/28858-val-isa-used-on-a-null-pointer.swift → validation-test/compiler_crashers_fixed/28858-val-isa-used-on-a-null-pointer.swift

@@ -6,7 +6,7 @@
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 
 // REQUIRES: asserts
-// RUN: not --crash %target-swift-frontend %s -emit-ir
+// RUN: not %target-swift-frontend %s -emit-ir
 func a
 class a<b}let d=a?as?CVarArg
 & a

validation-test/compiler_crashers/28863-t-is-archetypetype-t-isexistentialtype-expected-a-class-archetype-or-existential.swift → validation-test/compiler_crashers_fixed/28863-t-is-archetypetype-t-isexistentialtype-expected-a-class-archetype-or-existential.swift

@@ -6,7 +6,7 @@
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 
 // REQUIRES: asserts
-// RUN: not --crash %target-swift-frontend %s -emit-ir
+// RUN: not %target-swift-frontend %s -emit-ir
 class a
 class a<a{{}class a<a{var f=a
 & ManagedBuffer