Backport ASM 6.2 upgrade to 2.11.x via 2.12.x (scala#6733)

Avoid performance problem after ASM upgrade in prod/cons analysis

ASM 6.2 now creates a new Frame inside the loop in which
`newExceptionValue` is called. We were including this frame
in the case-class equality of the pseudo-instruction,
`ExceptionProducer`, and upon receiving new instances each
time the `ProdCons` analysis massively slowed down.

This commit just captures the data we need: the stack top of the
handler frame.

Upgrade to scala-asm 6.2

See: scala/scala-asm#5

Upstream changes in ASM:

  scala/scala-asm@ASM_6_0...ASM_6_2
  http://asm.ow2.io/versions.html

The motivations, other than just keeping current, are:

  - support for Java 9/10/11 updates to the classfile format.
  - reducing needless String => Array[Char] conversions thanks
    to internal changes in ASM.

This PR will fail to build until we publish artifact
from scala/scala-asm.

Includes a workaround for scala/bug#10418

Move to the standard way of defining a custom asm.Attribute

It seems we don't need CustomAttr in our fork of scala-asm,
we can just override Attribute.write.

Customise label handling without needing to modify ASM directly

Comment on our customizations to asm.tree.*Node

(cherry picked from commit 79b7f2a)

Author: retronym

src/compiler/scala/tools/nsc/backend/jvm/AsmUtils.scala

@@ -55,6 +55,48 @@ object AsmUtils {
     node
   }
 
+  def readClass(filename: String): ClassNode = readClass(classBytes(filename))
+
+  def classBytes(file: String): Array[Byte] = {
+    val f = new java.io.RandomAccessFile(file, "r")
+    val bytes = new Array[Byte](f.length.toInt)
+    f.read(bytes)
+    bytes
+  }
+
+  def classFromBytes(bytes: Array[Byte]): ClassNode = {
+    val node = new ClassNode1()
+    new ClassReader(bytes).accept(node, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES)
+
+    node
+  }
+
+//  def main(args: Array[String]): Unit = println(textify(sortedClassRead(classBytes(args.head))))
+
+  def sortClassMembers(node: ClassNode): node.type = {
+    node.fields.sort(_.name compareTo _.name)
+    node.methods.sort(_.name compareTo _.name)
+    node
+  }
+
+  // drop ScalaSig annotation and class attributes
+  def zapScalaClassAttrs(node: ClassNode): node.type = {
+    if (node.visibleAnnotations != null)
+      node.visibleAnnotations = node.visibleAnnotations.asScala.filterNot(a => a == null || a.desc.contains("Lscala/reflect/ScalaSignature")).asJava
+
+    node.attrs = null
+    node
+  }
+
+  def main(args: Array[String]): Unit = args.par.foreach { classFileName =>
+    val node = zapScalaClassAttrs(sortClassMembers(classFromBytes(classBytes(classFileName))))
+
+    val pw = new PrintWriter(classFileName + ".asm")
+    val trace = new TraceClassVisitor(pw)
+    node.accept(trace)
+    pw.close()
+  }
+
   /**
    * Returns a human-readable representation of the cnode ClassNode.
    */

src/compiler/scala/tools/nsc/backend/jvm/BCodeHelpers.scala

@@ -12,6 +12,7 @@ import scala.collection.mutable
 import scala.tools.nsc.io.AbstractFile
 import GenBCode._
 import BackendReporting._
+import scala.tools.asm.ClassWriter
 
 /*
  *  Traits encapsulating functionality to convert Scala AST Trees into ASM ClassNodes.
@@ -244,9 +245,14 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
      * can-multi-thread
      */
     def createJAttribute(name: String, b: Array[Byte], offset: Int, len: Int): asm.Attribute = {
-      val dest = new Array[Byte](len)
-      System.arraycopy(b, offset, dest, 0, len)
-      new asm.CustomAttr(name, dest)
+      new asm.Attribute(name) {
+        override def write(classWriter: ClassWriter, code: Array[Byte],
+                           codeLength: Int, maxStack: Int, maxLocals: Int): asm.ByteVector = {
+          val byteVector = new asm.ByteVector(len)
+          byteVector.putByteArray(b, offset, len)
+          byteVector
+        }
+      }
     }
 
     /*
@@ -766,7 +772,7 @@ abstract class BCodeHelpers extends BCodeIdiomatic with BytecodeWriters {
       this.cunit = cunit
 
       val bType = mirrorClassClassBType(moduleClass)
-      val mirrorClass = new asm.tree.ClassNode
+      val mirrorClass = new ClassNode1
       mirrorClass.visit(
         classfileVersion,
         bType.info.get.flags,

src/compiler/scala/tools/nsc/backend/jvm/BCodeSkelBuilder.scala

@@ -104,7 +104,7 @@ abstract class BCodeSkelBuilder extends BCodeHelpers {
 
       val classBType = classBTypeFromSymbol(claszSymbol)
 
-      cnode = new asm.tree.ClassNode()
+      cnode = new ClassNode1()
 
       initJClass(cnode)
 

src/compiler/scala/tools/nsc/backend/jvm/BTypes.scala

@@ -97,7 +97,7 @@ abstract class BTypes {
   /**
    * Obtain the BType for a type descriptor or internal name. For class descriptors, the ClassBType
    * is constructed by parsing the corresponding classfile.
-   * 
+   *
    * Some JVM operations use either a full descriptor or only an internal name. Example:
    *   ANEWARRAY java/lang/String    // a new array of strings (internal name for the String class)
    *   ANEWARRAY [Ljava/lang/String; // a new array of array of string (full descriptor for the String class)
@@ -964,6 +964,8 @@ abstract class BTypes {
             // finds the first common one.
             // MOST LIKELY the answer can be found here, see the comments and links by Miguel:
             //  - https://issues.scala-lang.org/browse/SI-3872
+            // @jz Wouldn't it be better to walk the superclass chain of both types in reverse (starting from Object), and
+            //     finding the last common link? That would be O(N), whereas this looks O(N^2)
             firstCommonSuffix(this :: this.superClassesTransitive.orThrow, other :: other.superClassesTransitive.orThrow)
         }
 
@@ -1155,4 +1157,4 @@ object BTypes {
   // no static way (without symbol table instance) to get to nme.ScalaATTR / ScalaSignatureATTR
   val ScalaAttributeName    = "Scala"
   val ScalaSigAttributeName = "ScalaSig"
-}
+}

src/compiler/scala/tools/nsc/backend/jvm/ClassNode1.java

@@ -0,0 +1,31 @@
+/* NSC -- new Scala compiler
+ * Copyright 2018 LAMP/EPFL
+ * @author Martin Odersky
+ */
+package scala.tools.nsc.backend.jvm;
+
+import scala.tools.asm.MethodVisitor;
+import scala.tools.asm.Opcodes;
+import scala.tools.asm.tree.ClassNode;
+import scala.tools.asm.tree.MethodNode;
+
+/**
+ * A subclass of {@link ClassNode} to customize the representation of
+ * label nodes with {@link LabelNode1}.
+ */
+public class ClassNode1 extends ClassNode {
+    public ClassNode1() {
+        this(Opcodes.ASM6);
+    }
+
+    public ClassNode1(int api) {
+        super(api);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        MethodNode method = new MethodNode1(access, name, descriptor, signature, exceptions);
+        methods.add(method);
+        return method;
+    }
+}

src/compiler/scala/tools/nsc/backend/jvm/LabelNode1.java

@@ -0,0 +1,23 @@
+/* NSC -- new Scala compiler
+ * Copyright 2018 LAMP/EPFL
+ * @author Martin Odersky
+ */
+package scala.tools.nsc.backend.jvm;
+
+import scala.tools.asm.Label;
+import scala.tools.asm.tree.ClassNode;
+import scala.tools.asm.tree.LabelNode;
+
+/**
+ * A subclass of {@link LabelNode} to add user-definable flags.
+ */
+public class LabelNode1 extends LabelNode {
+    public LabelNode1() {
+    }
+
+    public LabelNode1(Label label) {
+        super(label);
+    }
+
+    public int flags;
+}

src/compiler/scala/tools/nsc/backend/jvm/MethodNode1.java

@@ -0,0 +1,39 @@
+/* NSC -- new Scala compiler
+ * Copyright 2018 LAMP/EPFL
+ * @author Martin Odersky
+ */
+package scala.tools.nsc.backend.jvm;
+
+import scala.tools.asm.Label;
+import scala.tools.asm.Opcodes;
+import scala.tools.asm.tree.LabelNode;
+import scala.tools.asm.tree.MethodNode;
+/**
+ * A subclass of {@link MethodNode} to customize the representation of
+ * label nodes with {@link LabelNode1}.
+ */
+public class MethodNode1 extends MethodNode {
+    public MethodNode1(int api, int access, String name, String descriptor, String signature, String[] exceptions) {
+        super(api, access, name, descriptor, signature, exceptions);
+    }
+
+    public MethodNode1(int access, String name, String descriptor, String signature, String[] exceptions) {
+        this(Opcodes.ASM6, access, name, descriptor, signature, exceptions);
+    }
+
+    public MethodNode1(int api) {
+        super(api);
+    }
+
+    public MethodNode1() {
+        this(Opcodes.ASM6);
+    }
+
+    @Override
+    protected LabelNode getLabelNode(Label label) {
+        if (!(label.info instanceof LabelNode)) {
+            label.info = new LabelNode1(label);
+        }
+        return (LabelNode) label.info;
+    }
+}

src/compiler/scala/tools/nsc/backend/jvm/analysis/BackendUtils.scala

@@ -102,8 +102,13 @@ class ProdConsAnalyzer(methodNode: MethodNode, classInternalName: InternalName)
     inputValues(insn).iterator.flatMap(v => v.insns.asScala).toSet
   }
 
-  def consumersOfOutputsFrom(insn: AbstractInsnNode): Set[AbstractInsnNode] =
-    _consumersOfOutputsFrom.get(insn).map(v => v.indices.flatMap(v.apply)(collection.breakOut): Set[AbstractInsnNode]).getOrElse(Set.empty)
+  def consumersOfOutputsFrom(insn: AbstractInsnNode): Set[AbstractInsnNode] = insn match {
+    case _: UninitializedLocalProducer                      => Set.empty
+    case ParameterProducer(local)                           => consumersOfValueAt(methodNode.instructions.getFirst, local)
+    case ExceptionProducer(handlerLabel, handlerStackTop)   => consumersOfValueAt(handlerLabel, handlerStackTop)
+    case _ =>
+      _consumersOfOutputsFrom.get(insn).map(v => v.indices.flatMap(v.apply)(collection.breakOut): Set[AbstractInsnNode]).getOrElse(Set.empty)
+  }
 
   /**
    * Returns the potential initial producer instructions of a value in the frame of `insn`.
@@ -386,7 +391,7 @@ class ProdConsAnalyzer(methodNode: MethodNode, classInternalName: InternalName)
   private def outputValueSlots(insn: AbstractInsnNode): Seq[Int] = insn match {
     case ParameterProducer(local)          => Seq(local)
     case UninitializedLocalProducer(local) => Seq(local)
-    case ExceptionProducer(frame)          => Seq(frame.stackTop)
+    case ExceptionProducer(_, stackTop)    => Seq(stackTop)
     case _ =>
       if (insn.getOpcode == -1) return Seq.empty
       if (isStore(insn)) {
@@ -459,11 +464,11 @@ abstract class InitialProducer extends AbstractInsnNode(-1) {
   override def accept(cv: MethodVisitor): Unit = throw new UnsupportedOperationException
 }
 
-case class ParameterProducer(local: Int)                      extends InitialProducer
-case class UninitializedLocalProducer(local: Int)             extends InitialProducer
-case class ExceptionProducer(handlerFrame: Frame[_ <: Value]) extends InitialProducer
+case class ParameterProducer(local: Int)                                                extends InitialProducer
+case class UninitializedLocalProducer(local: Int)                                       extends InitialProducer
+case class ExceptionProducer[V <: Value](handlerLabel: LabelNode, handlerStackTop: Int) extends InitialProducer
 
-class InitialProducerSourceInterpreter extends SourceInterpreter {
+class InitialProducerSourceInterpreter extends SourceInterpreter(scala.tools.asm.Opcodes.ASM7_EXPERIMENTAL) {
   override def newParameterValue(isInstanceMethod: Boolean, local: Int, tp: Type): SourceValue = {
     new SourceValue(tp.getSize, ParameterProducer(local))
   }
@@ -473,6 +478,7 @@ class InitialProducerSourceInterpreter extends SourceInterpreter {
   }
 
   override def newExceptionValue(tryCatchBlockNode: TryCatchBlockNode, handlerFrame: Frame[_ <: Value], exceptionType: Type): SourceValue = {
-    new SourceValue(1, ExceptionProducer(handlerFrame))
+    val handlerStackTop = handlerFrame.stackTop + 1 // +1 because this value is about to be pushed onto `handlerFrame`.
+    new SourceValue(1, ExceptionProducer(tryCatchBlockNode.handler, handlerStackTop))
   }
 }

src/compiler/scala/tools/nsc/backend/jvm/analysis/ProdConsAnalyzer.scala

@@ -137,7 +137,7 @@ class ByteCodeRepository(val classPath: ClassFileLookup[AbstractFile], val isJav
   private def parseClass(internalName: InternalName): Either[ClassNotFound, ClassNode] = {
     val fullName = internalName.replace('/', '.')
     classPath.findClassFile(fullName) map { classFile =>
-      val classNode = new asm.tree.ClassNode()
+      val classNode = new ClassNode1
       val classReader = new asm.ClassReader(classFile.toByteArray)
 
       // Passing the InlineInfoAttributePrototype makes the ClassReader invoke the specific `read`

src/compiler/scala/tools/nsc/backend/jvm/opt/ByteCodeRepository.scala

@@ -283,7 +283,7 @@ object BytecodeUtils {
    */
   def newLabelNode: LabelNode = {
     val label = new Label
-    val labelNode = new LabelNode(label)
+    val labelNode = new LabelNode1(label)
     label.info = labelNode
     labelNode
   }

src/compiler/scala/tools/nsc/backend/jvm/opt/BytecodeUtils.scala

@@ -0,0 +1,61 @@
+/* NEST (New Scala Test)
+ * Copyright 2007-2013 LAMP/EPFL
+ * @author Paul Phillips
+ */
+package scala.tools.partest
+package nest
+
+import java.io.{Console => _, _}
+import java.nio.charset.Charset
+
+object StreamCapture {
+  def savingSystem[T](body: => T): T = {
+    val savedOut  = System.out
+    val savedErr  = System.err
+    try body
+    finally {
+      System setErr savedErr
+      System setOut savedOut
+    }
+  }
+
+  def capturingOutErr[A](output: OutputStream)(f: => A): A = {
+    import java.io._
+    val charset = Charset.defaultCharset()
+    val printStream = new PrintStream(output, true, charset.name())
+    savingSystem {
+      System.setOut(printStream)
+      System.setErr(printStream)
+      try {
+        scala.Console.withErr(printStream) {
+          scala.Console.withOut(printStream) {
+            f
+          }
+        }
+      } finally {
+        printStream.close()
+      }
+    }
+  }
+
+  def withExtraProperties[A](extra: Map[String, String])(action: => A): A = {
+    val saved = System.getProperties()
+    val modified = new java.util.Properties()
+    // on Java 9, we need to cast our way around this:
+    // src/main/scala/scala/tools/partest/nest/StreamCapture.scala:44: ambiguous reference to overloaded definition,
+    // both method putAll in class Properties of type (x$1: java.util.Map[_, _])Unit
+    // and  method putAll in class Hashtable of type (x$1: java.util.Map[_ <: Object, _ <: Object])Unit
+    // match argument types (java.util.Properties)
+    (modified: java.util.Hashtable[AnyRef, AnyRef]).putAll(saved)
+    extra.foreach { case (k, v) => modified.setProperty(k, v) }
+    // Trying to avoid other threads seeing the new properties object prior to the new entries
+    // https://github.com/scala/scala/pull/6391#issuecomment-371346171
+    UnsafeAccess.U.storeFence()
+    System.setProperties(modified)
+    try {
+      action
+    } finally {
+      System.setProperties(saved)
+    }
+  }
+}

src/partest/scala/tools/partest/nest/StreamCapture.scala

@@ -0,0 +1,43 @@
+package scala.tools.nsc
+package backend.jvm
+
+import java.util.concurrent.TimeUnit
+
+import scala.tools.asm.tree.ClassNode
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra.Blackhole
+
+import scala.collection.JavaConverters.asScalaIteratorConverter
+import scala.tools.asm.tree.ClassNode
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(2)
+@Threads(1)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+class ProdConsBenchmark {
+  type G <: Global
+  var global: G = _
+  private var classNode: ClassNode = _
+
+  @Setup(Level.Trial) def setup(): Unit = {
+    val settings = new Settings()
+    settings.usejavacp.value = true
+    val global = new Global(settings)
+    import global._
+    this.global = global.asInstanceOf[G]
+    classNode = AsmUtils.readClass(global.classPath.findClassFile("scala.tools.nsc.typechecker.Implicits$ImplicitSearch").get.toByteArray)
+  }
+
+  @Benchmark
+  def prodCons(bh: Blackhole): Unit = {
+    val global: G = this.global
+    import global.genBCode.postProcessor.backendUtils._
+    for (m <- classNode.methods.iterator().asScala) {
+      bh.consume(new ProdConsAnalyzer(m, classNode.name))
+    }
+  }
+}
+

test/benchmarks/src/main/scala/scala/tools/nsc/backend/jvm/ProdConsBenchmark.scala

@@ -33,7 +33,7 @@ scala-swing.version.number=1.0.2
 akka-actor.version.number=2.3.16
 actors-migration.version.number=1.1.0
 jline.version=2.14.6
-scala-asm.version=6.0.0-scala-1
+scala-asm.version=6.2.0-scala-2
 
 # external modules, used internally (not shipped)
 partest.version.number=1.0.16