Fix scala#8033: Eliminate unused outer accessors

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -101,7 +101,8 @@ class Compiler {
          new LazyVals,               // Expand lazy vals
          new Memoize,                // Add private fields to getters and setters
          new NonLocalReturns,        // Expand non-local returns
-         new CapturedVars) ::        // Represent vars captured by closures as heap objects
+         new CapturedVars,           // Represent vars captured by closures as heap objects
+         new ElimOuterAccessors) ::  // Drop unused outer accessors
     List(new Constructors,           // Collect initialization code in primary constructors
                                         // Note: constructors changes decls in transformTemplate, no InfoTransformers should be added after it
          new FunctionalInterfaces,   // Rewrites closures to implement @specialized types of Functions.

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1079,9 +1079,15 @@ object SymDenotations {
     final def lexicallyEnclosingClass(implicit ctx: Context): Symbol =
       if (!exists || isClass) symbol else owner.lexicallyEnclosingClass
 
+    /** A class is extensible if it is not final, nor a module class,
+     *  nor an anonymous class.
+     */
+    final def isExtensibleClass(using Context): Boolean =
+      isClass && !isOneOf(FinalOrModuleClass) && !isAnonymousClass
+
     /** A symbol is effectively final if it cannot be overridden in a subclass */
     final def isEffectivelyFinal(implicit ctx: Context): Boolean =
-      isOneOf(EffectivelyFinalFlags) || !owner.isClass || owner.isOneOf(FinalOrModuleClass) || owner.isAnonymousClass
+      isOneOf(EffectivelyFinalFlags) || !owner.isExtensibleClass
 
     /** A class is effectively sealed if has the `final` or `sealed` modifier, or it
      *  is defined in Scala 3 and is neither abstract nor open.

compiler/src/dotty/tools/dotc/transform/Constructors.scala

@@ -31,7 +31,7 @@ class Constructors extends MiniPhase with IdentityDenotTransformer { thisPhase =
 
   override def phaseName: String = Constructors.name
   override def runsAfter: Set[String] = Set(HoistSuperArgs.name)
-  override def runsAfterGroupsOf: Set[String] = Set(Memoize.name)
+  override def runsAfterGroupsOf: Set[String] = Set(Memoize.name, ElimOuterAccessors.name)
     // Memoized needs to be finished because we depend on the ownerchain after Memoize
     // when checking whether an ident is an access in a constructor or outside it.
     // This test is done in the right-hand side of a value definition. If Memoize

compiler/src/dotty/tools/dotc/transform/ElimOuterAccessors.scala

@@ -0,0 +1,53 @@
+package dotty.tools.dotc
+package transform
+
+import core._
+import MegaPhase.MiniPhase
+import dotty.tools.dotc.core.Contexts.Context
+import ast._
+import Trees._
+import Flags._
+import Symbols._
+import Decorators._
+import DenotTransformers._
+import collection.mutable
+
+object ElimOuterAccessors:
+  val name: String = "elimOuterAccessors"
+
+/** Drops outer accessors of final classes that are unused */
+class ElimOuterAccessors extends MiniPhase with IdentityDenotTransformer:
+  thisPhase =>
+  import tpd._
+
+  override def phaseName: String = ElimOuterAccessors.name
+
+  override def changesMembers: Boolean = true // the phase drops outer accessors
+
+  private def mightBeDropped(sym: Symbol)(using Context) =
+    sym.is(OuterAccessor) && !sym.owner.isExtensibleClass
+
+  /** The accessed outer accessors that might otherwise be dropped */
+  private val retainedAccessors = mutable.Set[Symbol]()
+
+  private def markAccessed(tree: RefTree)(implicit ctx: Context): Tree =
+    val sym = tree.symbol
+    if mightBeDropped(sym) then retainedAccessors += sym
+    tree
+
+  def transformStat(tree: Tree)(using Context): Tree =
+    val sym = tree.symbol
+    if mightBeDropped(sym) && !retainedAccessors.remove(sym) then
+      sym.dropAfter(thisPhase)
+      EmptyTree
+    else
+      tree
+
+  override def transformIdent(tree: Ident)(using Context): Tree =
+    markAccessed(tree)
+
+  override def transformSelect(tree: Select)(using Context): Tree =
+    markAccessed(tree)
+
+  override def transformTemplate(tree: Template)(using Context): Tree =
+    cpy.Template(tree)(body = tree.body.map(transformStat).filter(_ != EmptyTree))

tests/run/i8033.scala

@@ -0,0 +1,34 @@
+trait Okay extends Serializable {
+  def okay: Okay
+}
+
+class Foo {
+  def okay1: Okay = new Okay() {
+    val okay: Okay = this
+  }
+  def okay2: Okay = new Okay {
+    val okay: Okay = okay1
+  }
+}
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    val foo = new Foo
+    println(roundTrip(foo.okay1))
+    println(roundTrip(foo.okay2))
+  }
+
+  def roundTrip[A](a: A): A = {
+    import java.io._
+
+    val aos = new ByteArrayOutputStream()
+    val oos = new ObjectOutputStream(aos)
+    oos.writeObject(a)
+    oos.close()
+    val ais = new ByteArrayInputStream(aos.toByteArray())
+    val ois: ObjectInputStream = new ObjectInputStream(ais)
+    val newA = ois.readObject()
+    ois.close()
+    newA.asInstanceOf[A]
+  }
+}